Skip to content

Automate browser based workflows with AI

Task V3: continue a batched turn past a tool failure that did not move the page (#8309)

pedrohsdb96618fc406df
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 4bed3186b..0b2ac826e 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1777,16 +1777,31 @@ class ForgeAgent:
# the gate, not here. Built for both populations — the failure-evidence gate needs a sampler
# to run at all — over the same page accessor each population's tools use, so the evidence
# is sampled from the page the model actually acted on.
+ _PAGE_FINGERPRINT_PROBE_JS = (
+ "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"
+ " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"
+ " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"
+ )
+
+ _DOCUMENT_NONCE_JS = (
+ "() => { if (!window.__skyvern_doc_nonce) window.__skyvern_doc_nonce = String(Math.random());"
+ " return window.__skyvern_doc_nonce; }"
+ )
+
async def _page_fingerprint() -> str | None:
peek = await _fingerprint_page()
if peek is None:
return None
- probe_js = (
- "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"
- " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"
- " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"
- )
- return await peek.evaluate(probe_js)
+ return await peek.evaluate(_PAGE_FINGERPRINT_PROBE_JS)
+
+ # Document identity, not content: a failed call's leftover text or open menu changes the DOM
+ # without re-mapping other selectors, while a navigation or reload (which does) wipes the nonce.
+ async def _page_probe() -> str | None:
+ peek = await _fingerprint_page()
+ if peek is None:
+ return None
+ nonce = await peek.evaluate(_DOCUMENT_NONCE_JS)
+ return f"{peek.url}|{nonce}"
# Whether the control the run CLICKED is still in flight. Scoped to that one control on
# purpose: "is anything on this page busy?" strands a finished run on an unrelated upload
@@ -1834,6 +1849,7 @@ class ForgeAgent:
resolve_typed_text=resolve_typed_text,
page_free=page_free_validation,
page_fingerprint=_page_fingerprint,
+ page_probe=_page_probe,
# Unfenced across both populations, unlike the settle probe above: that fence exists
# to keep a RENDERING wait off the bare arm, and this asks a different question. The
# bare arm is where the measured specimen lives (SKY-14701 is what inheriting a fence
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 4090071fb..9dc519189 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -199,6 +199,7 @@ async def run_task_v3_agent_loop(
staged_downloads: set[str] | None = None,
verification_blocker: VerificationBlocker | None = None,
initial_navigation_status: int | None = None,
+ page_probe: Callable[[], Awaitable[str | None]] | None = None,
) -> LoopOutcome:
"""Run one Task V3 task to completion against `page`, returning the loop outcome.
@@ -208,7 +209,9 @@ async def run_task_v3_agent_loop(
`page_fingerprint` sampler is provided, a finish(completed) on an unsettled page IS forced back
for a bounded re-verification turn; without one, pre-finish re-verification is prompt guidance
only. `max_settle_deferrals=0` disables that completed-side re-verification while leaving the
- failure-evidence gate, which shares the sampler, intact."""
+ failure-evidence gate, which shares the sampler, intact. `page_probe` is a separate sampler (URL
+ plus fingerprint) the loop uses to detect whether a failed batched call moved the page; a
+ page-free run has no page to probe."""
# Presigned file URLs in the payload carry an HMAC token the model would otherwise have to
# retype verbatim into a tool call; masking them here and resolving inside the tool handlers
# (the same boundary credential placeholders already use) avoids that. Page-free runs have no
@@ -319,6 +322,7 @@ async def run_task_v3_agent_loop(
completion_probe=completion_probe,
staged_downloads=staged_downloads,
initial_navigation_status=initial_navigation_status,
+ page_probe=None if page_free else page_probe,
)
if refs.refs:
outcome.reason = refs.resolve(outcome.reason)
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index f5803303b..189ecd9aa 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -199,6 +199,10 @@ ACTION_LOOP_REASON_PREFIX = "action_loop:"
# capability failures, not dead-ends, and are left to the model / stay `failed`.
NAVIGATION_DEAD_END_STATUSES = frozenset({404, 410})
+# Defined here (not tools.py) so the batch-dispatch poisoning check below can compare against it
+# without an import cycle -- tools.py already imports ToolResult/ToolSpec from this module.
+PAGE_UNAVAILABLE_ERROR = "browser page unavailable"
+
# A navigation landed on a hard dead-end (HTTP 404/410): the target posting does not exist or was
# removed, so the goal cannot be completed there. Ends the run as `terminated`, matching v1's terminate
# verdict for the same condition. Covers both the in-loop `navigate` tool and the pre-loop initial-URL
@@ -467,23 +471,58 @@ FAILURE_EVIDENCE_MIN_TOOL_CALLS = 3
FAILURE_EVIDENCE_MIN_TURNS = 3
+def _is_enter_submit(tool_name: str, args: dict[str, Any]) -> bool:
+ """An Enter keypress or a type that pressed Enter: the submit shapes with no selector to key on."""
+ if tool_name == "press_key":
+ raw = str(args.get("key", ""))
+ # Space on a focused button activates it exactly like a click; a literal " " would strip to "".
+ if raw == " ":
+ return True
+ # Playwright accepts Modifier+Key chords, and Control/Meta+Enter is a real submit.
+ key = raw.strip().lower().rsplit("+", 1)[-1]
+ return key in ("enter", "return", "numpadenter", "space")
+ if tool_name == "type":
+ return bool(args.get("press_enter"))
+ return False
+
+
+def _call_selector(args: dict[str, Any]) -> str | None:
+ selector = args.get("selector")
+ if isinstance(selector, str) and selector:
+ return selector
+ mark = args.get("mark")
+ return f"mark={mark}" if isinstance(mark, (int, str)) else None
+
+
+_PAGE_PROBE_TIMEOUT_SECONDS = 10.0
+
+
+async def _sample_page_probe(page_probe: Callable[[], Awaitable[str | None]]) -> str | None:
+ """A raising or hung probe is as uninformative as a None one: all mean "no reading," not "unchanged."""
+ try:
+ return await asyncio.wait_for(page_probe(), timeout=_PAGE_PROBE_TIMEOUT_SECONDS)
+ except Exception:
+ return None
+
+
+def _may_submit(tool_name: str, args: dict[str, Any]) -> bool:
+ """A click, an Enter press, or a type that pressed Enter: the loop cannot tell a submit from any of them."""
+ return tool_name == "click" or _is_enter_submit(tool_name, args)
+
+
+def _is_finish(tool_name: str) -> bool:
+ return tool_name == "finish"
+
+
def _arms_failure_evidence(tool_name: str, args: dict[str, Any], ok: bool) -> bool:
"""solve_captcha arms on ANY dispatch — its "not solved" error is exactly the verdict the async
protocol can contradict. Other actions arm only when they reached the page AND in their
- submit-shaped form: any click, an Enter press, or a type that pressed Enter."""
+ submit-shaped form."""
if tool_name == "solve_captcha":
return True
if not ok:
return False
- if tool_name == "click":
- # Deliberately broad: the loop has no submit-classification for click targets, and
- # narrowing by label/selector text is the marker-parsing this design avoids.
- return True
- if tool_name == "press_key":
- return str(args.get("key", "")).strip().lower() in ("enter", "return", "numpadenter")
- if tool_name == "type":
- return bool(args.get("press_enter"))
- return False
+ return _may_submit(tool_name, args)
@dataclass
@@ -952,6 +991,7 @@ async def run_agent_tool_loop(
completion_probe: CompletionProbe | None = None,
staged_downloads: set[str] | None = None,
initial_navigation_status: int | None = None,
+ page_probe: Callable[[], Awaitable[str | None]] | None = None,
) -> LoopOutcome:
tool_by_name = {tool.name: tool for tool in tools}
# Per run, never logged: the hashes it keys are stable within this run (the only scope any guard
@@ -1146,6 +1186,12 @@ async def run_agent_tool_loop(
# navigate can clear it — the model is told to batch aggressively, and terminating on the first
# of a batched [navigate(dead), navigate(live)] would discard the recovery it planned.
pending_nav_dead_end: int | None = None
+ # A same-selector dependent of a failed page-action call is skipped; any later click, Enter-shaped
+ # submit, or finish in the batch is skipped too -- the loop cannot tell a submit from the first two,
+ # and a verdict written before the failure was seen may be wrong or mis-reasoned.
+ failed_selectors: set[str] = set()
+ batch_had_failure = False
+ marks_stale = False
for idx, (tool_call_id, tool_name, args) in enumerate(tool_calls):
# Enforce the cap per tool call so one batched turn cannot overrun it, and honor a
# cancellation that arrives mid-batch before the next click/type/submit runs. Neither
@@ -1159,6 +1205,58 @@ async def run_agent_tool_loop(
_append_skipped_tool_results(messages, tool_calls[idx:], "run canceled")
break
spec = tool_by_name.get(tool_name)
+ call_selector = _call_selector(args)
+ if marks_stale and call_selector is not None and call_selector.startswith("mark="):
+ messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "name": tool_name,
+ "content": (
+ "skipped: an earlier look in this batch renumbered the marks, so this mark was chosen "
+ "from an old screenshot; pick it again from the new one"
+ ),
+ }
+ )
+ continue
+ if call_selector is not None and call_selector in failed_selectors:
+ messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "name": tool_name,
+ "content": f"skipped: depends on an earlier call in this batch on {call_selector} that failed",
+ }
+ )
+ continue
+ if batch_had_failure and _is_finish(tool_name):
+ # Any verdict queued behind the failure was written before the model saw it: a completed
+ # one may be false, and a failed/terminated one carries a reason that predates the error.
+ messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "name": tool_name,
+ "content": (
+ "skipped: a field in this batch failed before this verdict was reached; "
+ "re-observe, then finish with a status that reflects the failure"
+ ),
+ }
+ )
+ continue
+ if batch_had_failure and _may_submit(tool_name, args):
+ messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "name": tool_name,
+ "content": (
+ "skipped: a field in this batch failed and this call may submit the form; "
+ "fix the field, then re-queue it"
+ ),
+ }
+ )
+ continue
# Once the action-step budget is spent, refuse a further page action — terminate, mirroring
# the step engine's max-steps stop — but let perception/finish through, since the cap bounds
# new action rounds, not the separate re-observe/finish turn the system prompt asks for.
@@ -1184,6 +1282,11 @@ async def run_agent_tool_loop(
await on_pre_action(tool_name, args)
except Exception:
LOG.warning("taskv3 on_pre_action callback failed", tool=tool_name, exc_info=True)
+ # Sampled before dispatch so an error below can be checked for having moved the page even
+ # when the tool set no flag (a raised exception carries none). Not billed to the tool's timing.
+ probe_before: str | None = None
+ if spec is not None and page_probe is not None:
+ probe_before = await _sample_page_probe(page_probe)
tool_started_at = time.monotonic()
if spec is None:
result = ToolResult.error(f"unknown_tool: {tool_name}")
@@ -1250,6 +1353,11 @@ async def run_agent_tool_loop(
if result.screenshots:
pending_screenshots = list(result.screenshots)
result_data = result.data or {}
+ if tool_name == "look" and result_data.get("marks_renumbered"):
+ # Every mark=N still queued in this batch was chosen before this look renumbered the
+ # marks, so it now names an arbitrary element; a look refused before rebuilding its
+ # manifest leaves the old marks (and their failed keys) live.
+ marks_stale = True
if progress is not None and observe_summary:
invalid_fields = observe_summary.get("invalid_fields")
if invalid_fields is not None and progress.observe(invalid_fields):
@@ -1438,11 +1546,34 @@ async def run_agent_tool_loop(
break
if result.status == "error":
- # A failed call can leave the page in a state the rest of this batch was not planned
- # against (e.g. a write after a failed navigate). Stop and skip the remaining calls so
- # the model re-plans next turn from the error rather than acting on a stale assumption.
- _append_skipped_tool_results(messages, tool_calls[idx + 1 :], "earlier tool call in this batch failed")
- break
+ # The rest of the batch is skipped only when the failed call moved the page: the tool's
+ # own signal, or the probe sampled before dispatch (a missing reading counts as moved).
+ poisoned = (
+ tool_name == "navigate"
+ or result.content == PAGE_UNAVAILABLE_ERROR
+ or bool(
+ result_data.get("page_transitioned")
+ or result_data.get("page_state_changed")
+ or result_data.get("navigation_dead_end")
+ )
+ )
+ # Whether the page moved is independent of the tool's kind: a wait that timed out because
+ # the site navigated poisons the batch just as a failed click would.
+ if not poisoned and spec is not None and page_probe is not None:
+ probe_after = await _sample_page_probe(page_probe)
+ poisoned = probe_before is None or probe_after is None or probe_after != probe_before
+ if poisoned:
+ _append_skipped_tool_results(
+ messages,
+ tool_calls[idx + 1 :],
+ "earlier tool call in this batch failed and changed the page — re-observe before "
+ "re-queuing these",
+ )
+ break
+ if spec is not None and (spec.billable or spec.recordable):
+ batch_had_failure = True
+ if call_selector is not None:
+ failed_selectors.add(call_selector)
# The batch settled on a dead page (an in-loop navigate hit a hard 404/410 and no later navigate
# recovered): end the run as terminated deterministically, matching v1, rather than leaving the
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 3beed2a6d..6ca31dffe 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -27,7 +27,13 @@ from PIL import Image, ImageDraw
from skyvern.constants import BROWSER_DOWNLOADING_SUFFIX
from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stem
from skyvern.forge.sdk.core.skyvern_context import URL_IN_TEXT, canonical_url, opaque_url_echo_window
-from skyvern.forge.taskv3.loop import NAVIGATION_DEAD_END_STATUSES, ToolHandler, ToolResult, ToolSpec
+from skyvern.forge.taskv3.loop import (
+ NAVIGATION_DEAD_END_STATUSES,
+ PAGE_UNAVAILABLE_ERROR,
+ ToolHandler,
+ ToolResult,
+ ToolSpec,
+)
from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_action
if TYPE_CHECKING:
@@ -41,8 +47,6 @@ LOG = structlog.get_logger()
# tab/popup is followed on the next call instead of leaving the loop stuck on a stale page.
PageProvider = Callable[[], Awaitable[Any]]
-PAGE_UNAVAILABLE_ERROR = "browser page unavailable"
-
# Cap on the page URL observe() echoes. Callers that register a secret URL for exact-match redaction
# must register this prefix too, or the truncated echo survives the scrub.
OBSERVE_URL_MAX_CHARS = 300
@@ -178,7 +182,9 @@ def _classify_commit(
"""Classify a value-must-change action from a before/after observable-state readback.
Ranked fail-closed: a readable did-not-commit is reported whatever the target re-resolved to, because
- only an error halts a batched turn (INV-1 guards the confident ok, not the refusal). A commit read off
+ an error halts the rest of a batched turn only when it moved the page -- otherwise the field is
+ reported unfilled and only its same-selector dependents and any later click or Enter are skipped
+ (INV-1 guards the confident ok, not the refusal). A commit read off
a target that re-resolved to n != 1 is `unverified` (INV-1); no readable committable state is
`unverified` (INV-2). `committed_value` hands in a caller's own value-dimension truth in place of the
generic any-field-changed rule.
@@ -4447,7 +4453,8 @@ def build_browser_tools(
if matches == 0:
return ToolResult.error(
f"{selector} no longer matches anything on the page — the page re-rendered since it was "
- "observed. Re-observe and act on fresh selectors from the new observation."
+ "observed. Re-observe and act on fresh selectors from the new observation.",
+ data={"page_state_changed": True},
)
return ToolResult.error(
f"{selector} matches {matches} elements, so it does not identify one control. Re-observe and "
@@ -4679,7 +4686,8 @@ def build_browser_tools(
return ToolResult.error(
f"{selector} no longer exists on the page — element markers vanish when the "
"page re-renders (a closed menu destroys its options). Re-observe and act on "
- "fresh selectors from the new observation."
+ "fresh selectors from the new observation.",
+ data={"page_state_changed": True},
)
# The re-attach may have been a re-render that cloned the row, so the count is re-read.
matches = await _marker_matches(page, selector)
@@ -4689,7 +4697,8 @@ def build_browser_tools(
return ToolResult.error(
f"{selector} now matches {matches} elements — the page re-rendered and cloned the "
"marked element, so the marker no longer identifies one control. Re-observe and act "
- "on fresh selectors from the new observation."
+ "on fresh selectors from the new observation.",
+ data={"page_state_changed": True},
)
else:
ambiguous = await _ambiguous_selector_error(page, selector)
@@ -4808,7 +4817,8 @@ def build_browser_tools(
await _clear_proxy_tags(page)
return ToolResult.error(
f"{selector} left the page before the click could land — it was replaced by a "
- "re-render; re-observe and act on fresh selectors"
+ "re-render; re-observe and act on fresh selectors",
+ data={"page_state_changed": True},
)
base = f"clicked {selector} (hidden native control, toggled directly) — now at {await _url(page)}"
else:
@@ -4830,10 +4840,13 @@ def build_browser_tools(
except Exception:
gone = False
if gone:
+ # A same-document re-render: the URL and document nonce read unchanged, so only this
+ # flag tells the loop the rest of the batch was planned against a stale page.
return ToolResult.error(
f"click on {selector} failed: the element no longer exists on the page — it was "
"likely removed by a re-render (e.g. a menu closed and destroyed its options). "
- f"Re-observe and act on fresh selectors. (original error: {type(e).__name__})"
+ f"Re-observe and act on fresh selectors. (original error: {type(e).__name__})",
+ data={"page_state_changed": True},
)
# Diagnosed only now, after the full actionability wait: a transient overlay (a toast,
# a closing menu) deserves the whole 15s to clear on its own, not a probe-shortened one.
@@ -5262,9 +5275,9 @@ def build_browser_tools(
f"clicked suggestion {opt_txt!r} for {selector} but it did not commit — the field is "
"NOT filled; re-observe and retry, do not proceed"
)
- # DID_NOT_COMMIT: a suggestion surfaced but the field did not accept it — the field is NOT
- # filled. Return an error so a batched turn halts here (the loop stops the rest of the batch
- # on error) instead of proceeding — e.g. to a queued submit — on an uncommitted field.
+ # DID_NOT_COMMIT: the field is NOT filled. The loop then skips any later click or Enter
+ # in the same batch -- it may be an unvalidated submit, and no production submit guard
+ # exists yet.
return ToolResult.error(
f"clicked suggestion {opt_txt!r} for {selector} but it did not commit — the field is NOT "
"filled; re-observe and retry, do not proceed"
@@ -5827,7 +5840,9 @@ def build_browser_tools(
# bind the element fresh immediately before uploading (missing now means it vanished mid-download).
el = await page.query_selector(selector)
if el is None:
- return ToolResult("error", f"no file input for selector {selector!r}", staged)
+ return ToolResult(
+ "error", f"no file input for selector {selector!r}", {**staged, "page_state_changed": True}
+ )
# Verify the upload took EFFECT, not just that set_input_files did not raise. Watch upload-like
# network dispatches across the set_input_files + settle window (the window we already dwell in,
# so this adds no latency); a genuine upload dispatches at least one, a silent no-op none.
@@ -5961,6 +5976,8 @@ def build_browser_tools(
except Exception:
pass
_look_manifest.clear()
+ # From here on the old marks are gone, whatever happens next -- the loop keys on this.
+ renumbered = {"marks_renumbered": True}
for e in elements:
n = int(e["n"])
try:
@@ -5983,10 +6000,11 @@ def build_browser_tools(
annotated = await asyncio.get_running_loop().run_in_executor(None, _annotate_screenshot, png, kept, vw)
except Exception as exc:
LOG.warning("taskv3 look annotation failed", exc_info=True)
- return ToolResult.error(f"look failed to render marks: {type(exc).__name__}: {exc}")
+ return ToolResult.error(f"look failed to render marks: {type(exc).__name__}: {exc}", data=renumbered)
if not kept:
return ToolResult.ok(
"look: no interactive controls are visible in the viewport. Scroll or re-observe.",
+ data=renumbered,
screenshots=[annotated],
)
@@ -6009,7 +6027,7 @@ def build_browser_tools(
if isinstance(data, dict) and data.get("truncated"):
header += f" (only the first {_LOOK_MAX_MARKS} are marked; scroll for more.)"
legend = header + "\n" + "\n".join(lines)
- return ToolResult.ok(legend, screenshots=[annotated])
+ return ToolResult.ok(legend, data=renumbered, screenshots=[annotated])
async def _resolve_mark(page: Any, mark: int) -> tuple[str | None, ToolResult | None]:
# Turn mark=N into a selector the existing click/type handlers act through. Resolution is the
@@ -6034,7 +6052,8 @@ def build_browser_tools(
if not connected:
return None, ToolResult.error(
f"mark {mark} no longer points to an element on the page — it moved or the page "
- "re-rendered since look(). Call look() again and act on a fresh number."
+ "re-rendered since look(). Call look() again and act on a fresh number.",
+ data={"page_state_changed": True},
)
return f'[data-tv3-act="{mark}"]', None
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 1483349eb..a906884d0 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -2537,12 +2537,14 @@ async def test_execute_task_v3_settle_completion_fenced_to_block_tasks(
monkeypatch, outcome, task_block=block, data_extraction_goal=None, extracted_information_schema=None
)
assert loop_mock.await_args.kwargs["page_fingerprint"] is not None
+ assert loop_mock.await_args.kwargs["page_probe"] is not None # batch-poisoning probe reaches the engine
assert loop_mock.await_args.kwargs["max_settle_deferrals"] > 0
_step, _task, bare_loop_mock, _post = await _run_execute_task_v3(
monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None
)
assert bare_loop_mock.await_args.kwargs["page_fingerprint"] is not None
+ assert bare_loop_mock.await_args.kwargs["page_probe"] is not None
assert bare_loop_mock.await_args.kwargs["max_settle_deferrals"] == 0
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index c1f43f67b..74ee475e0 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -15,7 +15,7 @@ _CANDIDATE_METHODS = frozenset(
)
_DISCOVERED_BROWSER_API_CALLS = {
- "skyvern/forge/agent.py": Counter({"evaluate": 1, "new_page": 1}),
+ "skyvern/forge/agent.py": Counter({"evaluate": 2, "new_page": 1}),
"skyvern/forge/agent_functions.py": Counter({"close": 1, "scroll_into_view_if_needed": 1}),
"skyvern/webeye/actions/handler.py": Counter(
{
@@ -71,8 +71,9 @@ _DISCOVERED_BROWSER_API_CALLS = {
}
_EVALUATE_CALLERS = {
- # Read-only DOM fingerprint sample for the v3 settle-before-complete check.
- "skyvern/forge/agent.py": Counter({"_page_fingerprint": 1}),
+ # Read-only DOM fingerprint sample for the v3 settle-before-complete check, and the per-document
+ # nonce the v3 loop reads to tell whether a failed batched call navigated the page.
+ "skyvern/forge/agent.py": Counter({"_page_fingerprint": 1, "_page_probe": 1}),
"skyvern/webeye/actions/handler.py": Counter(
{
"_blob_iframe_src_titles": 1,
@@ -213,21 +214,21 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 156
+ assert sum(sum(methods.values()) for methods in observed.values()) == 157
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 5
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 151
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 152
def test_every_raw_evaluate_call_is_classified() -> None:
observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}
assert observed == _EVALUATE_CALLERS
- assert sum(sum(callers.values()) for callers in observed.values()) == 25
+ assert sum(sum(callers.values()) for callers in observed.values()) == 26
def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 862979e8b..edfa19d89 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -461,6 +461,38 @@ async def test_engine_wires_failure_evidence_gate() -> None:
assert outcome.reason == "still blocked, re-verified"
+async def test_engine_forwards_the_page_probe_and_withholds_it_from_page_free_runs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Drop this kwarg anywhere on the way in and the loop classifies every unflagged error as
+ # non-poisoning -- the fail-open state -- with every loop-level test still green.
+ from skyvern.forge.taskv3 import engine as engine_mod
+ from skyvern.forge.taskv3.loop import LoopOutcome
+
+ captured: list[object] = []
+
+ async def _capture(**kwargs: object) -> LoopOutcome:
+ captured.append(kwargs.get("page_probe"))
+ return LoopOutcome(status="completed", reason="ok")
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)
+ await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()), llm_caller=_ScriptedCaller([]), goal="x", page_probe=probe
+ )
+ await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(_FakePage()),
+ llm_caller=_ScriptedCaller([]),
+ goal="x",
+ page_probe=probe,
+ page_free=True,
+ )
+ assert captured == [probe, None]
+
+
@pytest.mark.asyncio
async def test_engine_wires_the_pending_gate_and_withholds_it_from_page_free_runs(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index cd303fc38..9c3568bf7 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -24,6 +24,7 @@ from skyvern.exceptions import SkyvernContextWindowExceededError
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.taskv3 import loop as loop_module
from skyvern.forge.taskv3.loop import (
ACTION_LOOP_REASON_PREFIX,
FAILURE_EVIDENCE_MIN_TOOL_CALLS,
@@ -31,6 +32,7 @@ from skyvern.forge.taskv3.loop import (
NAV_DEAD_END_REASON_PREFIX,
NAVIGATION_DEAD_END_STATUSES,
NO_TOOL_CALL_NUDGE,
+ PAGE_UNAVAILABLE_ERROR,
PERCEPTION_STALL_NUDGE_AFTER,
PERCEPTION_STALL_REASON_PREFIX,
PERCEPTION_STALL_SHADOW_EVENT,
@@ -107,14 +109,44 @@ class _ScriptedCaller:
return {"choices": [{"message": message}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}
-def _recording_tool(name: str, sink: list[tuple[str, dict[str, Any]]], *, raises: bool = False) -> ToolSpec:
+def _recording_tool(
+ name: str, sink: list[tuple[str, dict[str, Any]]], *, raises: bool = False, billable: bool = False
+) -> ToolSpec:
async def handler(args: dict[str, Any]) -> ToolResult:
sink.append((name, args))
if raises:
raise RuntimeError("boom")
return ToolResult.ok(f"{name} done")
- return ToolSpec(name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler)
+ return ToolSpec(
+ name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler, billable=billable
+ )
+
+
+def _erroring_tool(
+ name: str,
+ sink: list[tuple[str, dict[str, Any]]],
+ *,
+ error_data: dict[str, Any] | None = None,
+ billable: bool = False,
+ recordable: bool = False,
+) -> ToolSpec:
+ """Like ``_recording_tool`` but returns ``ToolResult.error(...)`` directly (optionally carrying a
+ ``data`` payload, e.g. ``page_transitioned``) instead of raising -- ``raises=True`` on
+ ``_recording_tool`` only ever produces a bare ``tool_error: RuntimeError: boom`` with no data."""
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ sink.append((name, args))
+ return ToolResult.error(f"{name} failed", data=error_data)
+
+ return ToolSpec(
+ name=name,
+ description=name,
+ parameters={"type": "object", "properties": {}},
+ handler=handler,
+ billable=billable,
+ recordable=recordable,
+ )
async def _run(script: list[list[tuple[str, dict[str, Any]]]], tools: list[ToolSpec], **kwargs: Any):
@@ -167,13 +199,18 @@ async def test_perception_is_on_demand_never_injected() -> None:
assert len(click_calls) == 2
-def _look_tool(sink: list[tuple[str, dict[str, Any]]], *, image: bytes = b"\x89PNG-fake") -> ToolSpec:
+def _look_tool(
+ sink: list[tuple[str, dict[str, Any]]], *, image: bytes = b"\x89PNG-fake", fail_before_renumbering: bool = False
+) -> ToolSpec:
"""A `look`-shaped tool: returns a text legend AND a transient screenshot the loop must show the
- model on the next call only (never persisted to the transcript)."""
+ model on the next call only (never persisted to the transcript). Like the real tool it reports
+ `marks_renumbered` once the manifest was rebuilt; a failure before that point reports nothing."""
async def handler(args: dict[str, Any]) -> ToolResult:
sink.append(("look", args))
- return ToolResult.ok("[1] button 'Next'", screenshots=[image])
+ if fail_before_renumbering:
+ return ToolResult.error("look budget reached")
+ return ToolResult.ok("[1] button 'Next'", data={"marks_renumbered": True}, screenshots=[image])
return ToolSpec(
name="look",
@@ -678,14 +715,15 @@ async def test_tool_call_cap_stops_mid_batch() -> None:
@pytest.mark.asyncio
async def test_tool_error_stops_batch_and_skips_remaining() -> None:
- # A failed call mid-batch must stop the rest of the batch (so a later write can't run against a
- # page a failed earlier call left in a bad state) and answer the skipped calls, so the next turn
- # sees a valid transcript and re-plans from the error.
+ # A failed call whose own result signals a page transition must still stop the rest of the batch
+ # (so a later write can't run against a page the failed call left in a different state) and
+ # answer the skipped calls, so the next turn sees a valid transcript and re-plans from the error.
click_calls: list[tuple[str, dict[str, Any]]] = []
+ boom_calls: list[tuple[str, dict[str, Any]]] = []
type_calls: list[tuple[str, dict[str, Any]]] = []
tools = [
_recording_tool("click", click_calls),
- _recording_tool("boom", [], raises=True),
+ _erroring_tool("boom", boom_calls, error_data={"page_transitioned": True}),
_recording_tool("type", type_calls),
make_finish_tool(),
]
@@ -700,7 +738,634 @@ async def test_tool_error_stops_batch_and_skips_remaining() -> None:
assert len(type_calls) == 0 # the call after the error was skipped, not executed
turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
- assert any("tool_error: RuntimeError" in m["content"] for m in turn1_tool_msgs)
+ assert any(m.get("name") == "boom" and "boom failed" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_non_mutating_tool_error_lets_independent_batch_calls_run() -> None:
+ # A `type` failure that leaves the page unchanged (no page-transition data, probe reads
+ # unchanged) must not block unrelated select_option calls later in the same batch -- only a
+ # failure that may have left the page in an unplanned-for state should stop the batch. `type`
+ # and `select_option` are marked billable=True to match production (tools.py) so this exercises
+ # the real probe-gated branch, not a fixture shortcut that skips it.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ select_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ tools = [
+ _recording_tool("click", click_calls),
+ _recording_tool("type", type_calls, raises=True, billable=True),
+ _recording_tool("select_option", select_calls, billable=True),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("click", {"selector": "#ok"}),
+ ("type", {"selector": "#name"}),
+ ("select_option", {"selector": "#a"}),
+ ("select_option", {"selector": "#b"}),
+ ("select_option", {"selector": "#c"}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(select_calls) == 3 # all three ran despite the earlier, non-page-mutating error
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "tool_error" in m["content"] for m in turn1_tool_msgs)
+ assert not any("skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_failed_call_skips_only_same_selector_dependents() -> None:
+ # A failed `type` on "#q" should skip a later call that targets the SAME selector (it depends on
+ # the failed call having succeeded) but must not skip a call against an unrelated selector. All
+ # three tools are billable=True to match production (tools.py), with a constant probe so the
+ # probe-gated branch runs and reads unchanged.
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ press_calls: list[tuple[str, dict[str, Any]]] = []
+ select_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ tools = [
+ _recording_tool("type", type_calls, raises=True, billable=True),
+ _recording_tool("press_key", press_calls, billable=True),
+ _recording_tool("select_option", select_calls, billable=True),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("type", {"selector": "#q"}),
+ ("press_key", {"selector": "#q"}),
+ ("select_option", {"selector": "#z"}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(press_calls) == 0 # depends on the failed #q type, must not run
+ assert len(select_calls) == 1 # unrelated selector, must run
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(
+ m.get("name") == "press_key" and "skipped" in m["content"] and "#q" in m["content"] for m in turn1_tool_msgs
+ )
+
+
+async def test_failed_call_skips_only_same_mark_dependents() -> None:
+ # A failed `type(mark=3)` should skip a later call on the SAME mark, mirroring the selector case,
+ # since act-by-mark calls carry no top-level "selector" arg for _call_selector to key on. The
+ # dependents are selects, not clicks: a click after any batch failure is deferred as a possible submit.
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ select_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [
+ _recording_tool("type", type_calls, raises=True, billable=True),
+ _recording_tool("select_option", select_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("type", {"mark": 3}),
+ ("select_option", {"mark": 3}),
+ ("select_option", {"mark": 4}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ assert outcome.status == "completed"
+ assert len(select_calls) == 1 # only mark=4 dispatched; mark=3 depends on the failed type
+ assert select_calls[0][1]["mark"] == 4
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "select_option" and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_mid_batch_look_defers_every_later_mark_call() -> None:
+ # look() renumbers marks on every call, so a mark=3 queued behind a mid-batch look was chosen from
+ # the OLD screenshot and now names an arbitrary element: it is deferred, not dispatched.
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ look_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ async def type_handler(args: dict[str, Any]) -> ToolResult:
+ type_calls.append(("type", args))
+ if len(type_calls) == 1:
+ return ToolResult.error("type failed")
+ return ToolResult.ok("type done")
+
+ tools = [
+ ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=type_handler,
+ billable=True,
+ ),
+ _look_tool(look_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("type", {"mark": 3}),
+ ("look", {}),
+ ("type", {"mark": 3}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(look_calls) == 1
+ assert len(type_calls) == 1 # the second mark=3 is deferred: its number predates the renumbering
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "type"]
+ assert any("renumbered" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_non_page_action_failure_does_not_mark_its_selector_or_arm_the_batch() -> None:
+ # A timed-out wait on #x mutates nothing: the later click on #x re-resolves the element itself, so
+ # it must dispatch rather than be skipped as a dependent, and no submit deferral is armed.
+ wait_calls: list[tuple[str, dict[str, Any]]] = []
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ async def wait_handler(args: dict[str, Any]) -> ToolResult:
+ wait_calls.append(("wait", args))
+ return ToolResult.error("wait timed out")
+
+ tools = [
+ ToolSpec(
+ name="wait", description="wait", parameters={"type": "object", "properties": {}}, handler=wait_handler
+ ),
+ _recording_tool("click", click_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("wait", {"selector": "#x"}), ("click", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "done"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(wait_calls) == 1
+ assert len(click_calls) == 1
+
+
+async def test_non_billable_failure_that_moved_the_page_still_stops_the_batch() -> None:
+ # A wait that times out BECAUSE the site navigated is not a field failure, but the page moved: the
+ # probe runs around every known tool, so the rest of the batch (planned for the old page) is skipped.
+ readings = iter(["doc-1", "doc-2"])
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return next(readings, "doc-2")
+
+ async def wait_handler(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.error("wait timed out")
+
+ tools = [
+ ToolSpec(
+ name="wait", description="wait", parameters={"type": "object", "properties": {}}, handler=wait_handler
+ ),
+ _recording_tool("click", click_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("wait", {"selector": "#x"}), ("click", {"selector": "#next"})],
+ [("finish", {"status": "terminated", "reason": "gave up"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "terminated"
+ assert click_calls == []
+ click_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "click"]
+ assert any("changed the page" in m["content"] for m in click_msgs)
+
+
+async def test_hung_page_probe_is_bounded_and_reads_as_poisoned(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A renderer that never answers the probe must not stall the loop past its deadline: the sample is
+ # bounded, and a missing reading is treated as poisoned (the batch stops), never as unchanged.
+ monkeypatch.setattr(loop_module, "_PAGE_PROBE_TIMEOUT_SECONDS", 0.01)
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def hung_probe() -> str | None:
+ await asyncio.Event().wait()
+ return None
+
+ async def type_handler(args: dict[str, Any]) -> ToolResult:
+ type_calls.append(("type", args))
+ if args.get("selector") == "#q":
+ return ToolResult.error("type failed")
+ return ToolResult.ok("type done")
+
+ tools = [
+ ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=type_handler,
+ billable=True,
+ ),
+ make_finish_tool(),
+ ]
+ script = [
+ [("type", {"selector": "#q"}), ("type", {"selector": "#zip"})],
+ [("finish", {"status": "terminated", "reason": "gave up"})],
+ ]
+ outcome, _ = await asyncio.wait_for(_run(script, tools, page_probe=hung_probe), timeout=2)
+
+ assert outcome.status == "terminated"
+ assert [call_args.get("selector") for _, call_args in type_calls] == ["#q"] # batch stopped: reading missing
+
+
+async def test_look_that_fails_before_renumbering_keeps_mark_dependents() -> None:
+ # A look() refused on budget (or failing to capture/enumerate) leaves the old manifest live, so a
+ # mark=3 call after it still names the element whose earlier call failed and must stay skipped.
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ look_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ async def type_handler(args: dict[str, Any]) -> ToolResult:
+ type_calls.append(("type", args))
+ return ToolResult.error("type failed")
+
+ tools = [
+ ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=type_handler,
+ billable=True,
+ ),
+ _look_tool(look_calls, fail_before_renumbering=True),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("type", {"mark": 3}),
+ ("look", {}),
+ ("type", {"mark": 3}),
+ ],
+ [("finish", {"status": "terminated", "reason": "gave up"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "terminated"
+ assert len(look_calls) == 1
+ assert len(type_calls) == 1 # the second mark=3 call is still a dependent of the failed one
+ type_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "type"]
+ assert any("skipped" in m["content"] for m in type_msgs)
+
+
+ ("finish_status", "expected_turns", "expected_status"),
+ [
+ ("completed", 2, "completed"),
+ ("terminated", 2, "terminated"),
+ ("failed", 2, "failed"),
+ ],
+)
+async def test_any_finish_deferred_after_batch_failure(
+ finish_status: str, expected_turns: int, expected_status: str
+) -> None:
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ async def type_handler(args: dict[str, Any]) -> ToolResult:
+ type_calls.append(("type", args))
+ return ToolResult.error("type failed")
+
+ tools = [
+ ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=type_handler,
+ billable=True,
+ ),
+ make_finish_tool(),
+ ]
+ script = [
+ [("type", {"selector": "#q"}), ("finish", {"status": finish_status, "reason": "done"})],
+ [("finish", {"status": finish_status, "reason": "done after re-checking"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ # Every verdict was written before the model saw the failure, so each is deferred one turn: a
+ # completed one may be false, and a failed/terminated one carries a reason that predates the error.
+ assert outcome.status == expected_status
+ assert outcome.turns == expected_turns
+ finish_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]
+ assert any("skipped" in m["content"] for m in finish_msgs)
+
+
+async def test_cross_selector_dependent_call_still_dispatches_and_fails_on_its_own() -> None:
+ # A call against a DIFFERENT selector than the failed one is not skipped by the same-selector
+ # rule -- it dispatches and, if it truly depends on the failed call's DOM effect, fails on its
+ # own terms rather than being wrong-committed as "skipped".
+ calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ calls.append(("select_option", args))
+ return ToolResult.error(f"no element for selector {args['selector']!r}")
+
+ tools = [
+ ToolSpec(
+ name="select_option", description="s", parameters={"type": "object", "properties": {}}, handler=handler
+ ),
+ make_finish_tool(),
+ ]
+ script = [
+ [("select_option", {"selector": "#a"}), ("select_option", {"selector": "#b"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ assert outcome.status == "completed"
+ assert len(calls) == 2 # #b was dispatched -- its selector differs from #a's, so it is not skipped
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "select_option"]
+ assert all("skipped" not in m["content"] for m in turn1_tool_msgs)
+ assert any("no element for selector '#b'" in m["content"] for m in turn1_tool_msgs)
+
+
+ "submit_call",
+ [
+ ("press_key", {"key": "Enter"}),
+ ("press_key", {"key": "Control+Enter"}),
+ ("press_key", {"selector": "#submit", "key": "Space"}),
+ ("press_key", {"selector": "#submit", "key": " "}),
+ ("type", {"selector": "#other", "press_enter": True}),
+ ],
+ ids=[
+ "press_key_enter",
+ "press_key_control_enter",
+ "press_key_space",
+ "press_key_literal_space",
+ "type_press_enter",
+ ],
+)
+async def test_click_and_enter_submit_skipped_after_batch_failure_but_other_fields_run(
+ submit_call: tuple[str, dict[str, Any]],
+) -> None:
+ # After a page-action failure in the batch, the loop cannot classify a click -- it may be the
+ # form's Submit -- so ANY later click is skipped alongside the Enter-shaped submit shapes. Other
+ # field-filling tools (select_combobox, type, file_upload) on OTHER selectors are not submit-shaped
+ # and still run.
+ submit_name, submit_args = submit_call
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ press_calls: list[tuple[str, dict[str, Any]]] = []
+ combobox_calls: list[tuple[str, dict[str, Any]]] = []
+ upload_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "doc-1"
+
+ async def type_handler(args: dict[str, Any]) -> ToolResult:
+ type_calls.append(("type", args))
+ if args.get("selector") == "#q":
+ return ToolResult.error("type failed")
+ return ToolResult.ok("type done")
+
+ tools = [
+ ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=type_handler,
+ billable=True,
+ ),
+ _recording_tool("click", click_calls),
+ _recording_tool("press_key", press_calls),
+ _recording_tool("select_combobox", combobox_calls),
+ _recording_tool("file_upload", upload_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("type", {"selector": "#q"}),
+ ("click", {"selector": "#agree"}),
+ (submit_name, submit_args),
+ ("select_combobox", {"selector": "#city"}),
+ ("type", {"selector": "#zip"}),
+ ("file_upload", {"selector": "#resume"}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(click_calls) == 0 # a click cannot be classified as safe, so it's skipped too
+ if submit_name == "press_key":
+ assert len(press_calls) == 0 # Enter-shaped submit skipped after the batch failure
+ else:
+ # The submit-shaped `type` call is skipped before dispatch -- only the earlier, failed "#q"
+ # call and the later "#zip" call reach the handler and land in the sink.
+ assert not any(call_args.get("selector") == "#other" for _, call_args in type_calls)
+ assert len(combobox_calls) == 1 # unrelated field, not submit-shaped, still runs
+ assert any(call_args.get("selector") == "#zip" for _, call_args in type_calls) # unrelated type still runs
+ assert len(upload_calls) == 1 # unrelated field, not submit-shaped, still runs
+ assert outcome.tool_calls == 5 # four dispatched calls plus finish: the two skipped calls cost no budget
+
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "click" and "skipped" in m["content"] for m in turn1_tool_msgs)
+ assert any(m.get("name") == submit_name and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+ ("data_flag", "error_value", "expected_status"),
+ [
+ ("page_transitioned", True, "completed"),
+ ("page_state_changed", True, "completed"),
+ # navigation_dead_end additionally classifies the run as terminated once the batch settles --
+ # that's a separate mechanism from the batch-stop this test targets, so it gets its own expected
+ # final status rather than "completed".
+ ("navigation_dead_end", 404, "terminated"),
+ ],
+)
+async def test_page_changing_tool_error_still_stops_batch(
+ data_flag: str, error_value: Any, expected_status: str
+) -> None:
+ # An error result that itself signals the page moved (page_transitioned, page_state_changed, or
+ # navigation_dead_end in .data) must still stop the rest of the batch even though the call
+ # "failed" -- the page moved out from under any planned follow-up regardless of the reported status.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ nav_click_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [
+ _recording_tool("click", click_calls),
+ _erroring_tool("nav_click", nav_click_calls, error_data={data_flag: error_value}),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [
+ ("click", {"selector": "#ok"}),
+ ("nav_click", {"selector": "#nav"}),
+ ("type", {"selector": "#x"}),
+ ],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ assert outcome.status == expected_status
+ assert len(type_calls) == 0 # the batch stopped: the page moved under the failed call
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_navigate_tool_error_still_stops_batch() -> None:
+ # A failed `navigate` carries no explicit page_transitioned/page_state_changed data, but
+ # navigation is inherently page-mutating -- an errored navigate must still stop the batch.
+ navigate_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [
+ _recording_tool("navigate", navigate_calls, raises=True),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("navigate", {"url": "https://example.com"}), ("type", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ assert outcome.status == "completed"
+ assert len(type_calls) == 0
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_page_unavailable_tool_error_still_stops_batch() -> None:
+ # The page itself is gone: inherently poisoning regardless of tool name or data.
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.error(PAGE_UNAVAILABLE_ERROR)
+
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [
+ ToolSpec(name="click", description="c", parameters={"type": "object", "properties": {}}, handler=handler),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ assert outcome.status == "completed"
+ assert len(type_calls) == 0
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_page_probe_change_across_failed_call_stops_batch() -> None:
+ # A billable tool's error carries no data flag, but the page_probe sampled before and after the
+ # dispatch shows the page changed underneath it -- still poisoning.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ probe_calls = {"n": 0}
+
+ async def probe() -> str | None:
+ probe_calls["n"] += 1
+ return "A" if probe_calls["n"] == 1 else "B"
+
+ tools = [
+ _erroring_tool("click", click_calls, billable=True),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(type_calls) == 0 # the probe changed across the failed call -- batch stopped
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
+
+
+async def test_page_probe_unchanged_across_failed_call_continues_batch() -> None:
+ # Same shape as above, but the probe reads the same value before and after the failed call --
+ # nothing in this error signals a page change, so the batch continues.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+
+ async def probe() -> str | None:
+ return "same"
+
+ tools = [
+ _erroring_tool("click", click_calls, billable=True),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(type_calls) == 1 # the probe read unchanged across the failed call -- batch continues
+
+
+async def test_recordable_non_billable_tool_error_with_probe_change_stops_batch() -> None:
+ # solve_captcha is recordable but not billable -- the probe must still be sampled around it, or a
+ # failed solve that moved the page can never poison the batch.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ type_calls: list[tuple[str, dict[str, Any]]] = []
+ probe_calls = {"n": 0}
+
+ async def probe() -> str | None:
+ probe_calls["n"] += 1
+ return "doc-1" if probe_calls["n"] == 1 else "doc-2"
+
+ tools = [
+ _erroring_tool("solve_captcha", click_calls, recordable=True),
+ _recording_tool("type", type_calls),
+ make_finish_tool(),
+ ]
+ script = [
+ [("solve_captcha", {}), ("type", {"selector": "#x"})],
+ [("finish", {"status": "completed", "reason": "recovered"})],
+ ]
+ outcome, _ = await _run(script, tools, page_probe=probe)
+
+ assert outcome.status == "completed"
+ assert len(type_calls) == 0 # the probe changed across the failed recordable call -- batch stopped
+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]
+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)
@pytest.mark.asyncio
@@ -1231,7 +1896,9 @@ async def test_every_executed_tool_call_emits_one_timing_record() -> None:
# A null selector is the case that matters: the tools fall back to scanning the whole page,
# so it must read as absent even though the key is present.
[("observe", {"selector": "sel"}), ("click", {"selector": None})],
- [("boom", {}), ("click", {})], # boom fails, so the trailing click is skipped, not executed
+ # boom fails on "#s"; the trailing click on the SAME selector depends on it and is skipped
+ # before dispatch (SKY-15143: a non-page-mutating error no longer halts the whole batch).
+ [("boom", {"selector": "#s"}), ("click", {"selector": "#s"})],
[("nope", {})], # unknown tool
[("finish", {"status": "completed", "reason": "ok"})],
]
@@ -1248,7 +1915,7 @@ async def test_every_executed_tool_call_emits_one_timing_record() -> None:
assert [entry["tool_status"] for entry in records] == ["ok", "ok", "error", "error", "ok"]
assert [entry["batch_size"] for entry in records] == [2, 2, 2, 1, 1]
assert [entry["batch_index"] for entry in records] == [0, 1, 0, 0, 0]
- assert [entry["selector_present"] for entry in records] == [True, False, False, False, False]
+ assert [entry["selector_present"] for entry in records] == [True, False, True, False, False]
assert [entry["billable"] for entry in records] == [False, False, True, False, False]
assert [entry["turn"] for entry in records] == [1, 1, 2, 3, 4]
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index d2fb597d5..5c13d7d56 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -5163,6 +5163,7 @@ async def test_click_on_a_marker_the_page_cloned_never_silently_lands_on_the_clo
assert clicked == [], f"failed loud yet still dispatched a click on {clicked}"
assert "e-observe" in r.content, r.content
assert selector in r.content
+ assert r.data == {"page_state_changed": True} # a cloning re-render must poison the batch
@_skip_no_browser
@@ -5212,6 +5213,7 @@ async def test_click_on_a_marker_the_page_destroyed_still_fails_loud() -> None:
assert r.status == "error"
assert "no longer exists" in r.content and "e-observe" in r.content
+ assert r.data == {"page_state_changed": True} # a same-document re-render must poison the batch
assert await page.evaluate("() => window.__clicked") == []