Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task V3: continue a batched turn past a tool failure that did not move the page (#8309)
pedrohsdb96618fc406df
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 4bed3186b..0b2ac826e 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1777,16 +1777,31 @@ class ForgeAgent:# the gate, not here. Built for both populations — the failure-evidence gate needs a sampler# to run at all — over the same page accessor each population's tools use, so the evidence# is sampled from the page the model actually acted on.+ _PAGE_FINGERPRINT_PROBE_JS = (+ "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"+ " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"+ " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"+ )++ _DOCUMENT_NONCE_JS = (+ "() => { if (!window.__skyvern_doc_nonce) window.__skyvern_doc_nonce = String(Math.random());"+ " return window.__skyvern_doc_nonce; }"+ )+async def _page_fingerprint() -> str | None:peek = await _fingerprint_page()if peek is None:return None- probe_js = (- "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"- " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"- " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"- )- return await peek.evaluate(probe_js)+ return await peek.evaluate(_PAGE_FINGERPRINT_PROBE_JS)++ # Document identity, not content: a failed call's leftover text or open menu changes the DOM+ # without re-mapping other selectors, while a navigation or reload (which does) wipes the nonce.+ async def _page_probe() -> str | None:+ peek = await _fingerprint_page()+ if peek is None:+ return None+ nonce = await peek.evaluate(_DOCUMENT_NONCE_JS)+ return f"{peek.url}|{nonce}"# Whether the control the run CLICKED is still in flight. Scoped to that one control on# purpose: "is anything on this page busy?" strands a finished run on an unrelated upload@@ -1834,6 +1849,7 @@ class ForgeAgent:resolve_typed_text=resolve_typed_text,page_free=page_free_validation,page_fingerprint=_page_fingerprint,+ page_probe=_page_probe,# Unfenced across both populations, unlike the settle probe above: that fence exists# to keep a RENDERING wait off the bare arm, and this asks a different question. The# bare arm is where the measured specimen lives (SKY-14701 is what inheriting a fencediff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex 4090071fb..9dc519189 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -199,6 +199,7 @@ async def run_task_v3_agent_loop(staged_downloads: set[str] | None = None,verification_blocker: VerificationBlocker | None = None,initial_navigation_status: int | None = None,+ page_probe: Callable[[], Awaitable[str | None]] | None = None,) -> LoopOutcome:"""Run one Task V3 task to completion against `page`, returning the loop outcome.@@ -208,7 +209,9 @@ async def run_task_v3_agent_loop(`page_fingerprint` sampler is provided, a finish(completed) on an unsettled page IS forced backfor a bounded re-verification turn; without one, pre-finish re-verification is prompt guidanceonly. `max_settle_deferrals=0` disables that completed-side re-verification while leaving the- failure-evidence gate, which shares the sampler, intact."""+ failure-evidence gate, which shares the sampler, intact. `page_probe` is a separate sampler (URL+ plus fingerprint) the loop uses to detect whether a failed batched call moved the page; a+ page-free run has no page to probe."""# Presigned file URLs in the payload carry an HMAC token the model would otherwise have to# retype verbatim into a tool call; masking them here and resolving inside the tool handlers# (the same boundary credential placeholders already use) avoids that. Page-free runs have no@@ -319,6 +322,7 @@ async def run_task_v3_agent_loop(completion_probe=completion_probe,staged_downloads=staged_downloads,initial_navigation_status=initial_navigation_status,+ page_probe=None if page_free else page_probe,)if refs.refs:outcome.reason = refs.resolve(outcome.reason)diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex f5803303b..189ecd9aa 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -199,6 +199,10 @@ ACTION_LOOP_REASON_PREFIX = "action_loop:"# capability failures, not dead-ends, and are left to the model / stay `failed`.NAVIGATION_DEAD_END_STATUSES = frozenset({404, 410})+# Defined here (not tools.py) so the batch-dispatch poisoning check below can compare against it+# without an import cycle -- tools.py already imports ToolResult/ToolSpec from this module.+PAGE_UNAVAILABLE_ERROR = "browser page unavailable"+# A navigation landed on a hard dead-end (HTTP 404/410): the target posting does not exist or was# removed, so the goal cannot be completed there. Ends the run as `terminated`, matching v1's terminate# verdict for the same condition. Covers both the in-loop `navigate` tool and the pre-loop initial-URL@@ -467,23 +471,58 @@ FAILURE_EVIDENCE_MIN_TOOL_CALLS = 3FAILURE_EVIDENCE_MIN_TURNS = 3+def _is_enter_submit(tool_name: str, args: dict[str, Any]) -> bool:+ """An Enter keypress or a type that pressed Enter: the submit shapes with no selector to key on."""+ if tool_name == "press_key":+ raw = str(args.get("key", ""))+ # Space on a focused button activates it exactly like a click; a literal " " would strip to "".+ if raw == " ":+ return True+ # Playwright accepts Modifier+Key chords, and Control/Meta+Enter is a real submit.+ key = raw.strip().lower().rsplit("+", 1)[-1]+ return key in ("enter", "return", "numpadenter", "space")+ if tool_name == "type":+ return bool(args.get("press_enter"))+ return False+++def _call_selector(args: dict[str, Any]) -> str | None:+ selector = args.get("selector")+ if isinstance(selector, str) and selector:+ return selector+ mark = args.get("mark")+ return f"mark={mark}" if isinstance(mark, (int, str)) else None+++_PAGE_PROBE_TIMEOUT_SECONDS = 10.0+++async def _sample_page_probe(page_probe: Callable[[], Awaitable[str | None]]) -> str | None:+ """A raising or hung probe is as uninformative as a None one: all mean "no reading," not "unchanged."""+ try:+ return await asyncio.wait_for(page_probe(), timeout=_PAGE_PROBE_TIMEOUT_SECONDS)+ except Exception:+ return None+++def _may_submit(tool_name: str, args: dict[str, Any]) -> bool:+ """A click, an Enter press, or a type that pressed Enter: the loop cannot tell a submit from any of them."""+ return tool_name == "click" or _is_enter_submit(tool_name, args)+++def _is_finish(tool_name: str) -> bool:+ return tool_name == "finish"++def _arms_failure_evidence(tool_name: str, args: dict[str, Any], ok: bool) -> bool:"""solve_captcha arms on ANY dispatch — its "not solved" error is exactly the verdict the asyncprotocol can contradict. Other actions arm only when they reached the page AND in their- submit-shaped form: any click, an Enter press, or a type that pressed Enter."""+ submit-shaped form."""if tool_name == "solve_captcha":return Trueif not ok:return False- if tool_name == "click":- # Deliberately broad: the loop has no submit-classification for click targets, and- # narrowing by label/selector text is the marker-parsing this design avoids.- return True- if tool_name == "press_key":- return str(args.get("key", "")).strip().lower() in ("enter", "return", "numpadenter")- if tool_name == "type":- return bool(args.get("press_enter"))- return False+ return _may_submit(tool_name, args)@dataclass@@ -952,6 +991,7 @@ async def run_agent_tool_loop(completion_probe: CompletionProbe | None = None,staged_downloads: set[str] | None = None,initial_navigation_status: int | None = None,+ page_probe: Callable[[], Awaitable[str | None]] | None = None,) -> LoopOutcome:tool_by_name = {tool.name: tool for tool in tools}# Per run, never logged: the hashes it keys are stable within this run (the only scope any guard@@ -1146,6 +1186,12 @@ async def run_agent_tool_loop(# navigate can clear it — the model is told to batch aggressively, and terminating on the first# of a batched [navigate(dead), navigate(live)] would discard the recovery it planned.pending_nav_dead_end: int | None = None+ # A same-selector dependent of a failed page-action call is skipped; any later click, Enter-shaped+ # submit, or finish in the batch is skipped too -- the loop cannot tell a submit from the first two,+ # and a verdict written before the failure was seen may be wrong or mis-reasoned.+ failed_selectors: set[str] = set()+ batch_had_failure = False+ marks_stale = Falsefor idx, (tool_call_id, tool_name, args) in enumerate(tool_calls):# Enforce the cap per tool call so one batched turn cannot overrun it, and honor a# cancellation that arrives mid-batch before the next click/type/submit runs. Neither@@ -1159,6 +1205,58 @@ async def run_agent_tool_loop(_append_skipped_tool_results(messages, tool_calls[idx:], "run canceled")breakspec = tool_by_name.get(tool_name)+ call_selector = _call_selector(args)+ if marks_stale and call_selector is not None and call_selector.startswith("mark="):+ messages.append(+ {+ "role": "tool",+ "tool_call_id": tool_call_id,+ "name": tool_name,+ "content": (+ "skipped: an earlier look in this batch renumbered the marks, so this mark was chosen "+ "from an old screenshot; pick it again from the new one"+ ),+ }+ )+ continue+ if call_selector is not None and call_selector in failed_selectors:+ messages.append(+ {+ "role": "tool",+ "tool_call_id": tool_call_id,+ "name": tool_name,+ "content": f"skipped: depends on an earlier call in this batch on {call_selector} that failed",+ }+ )+ continue+ if batch_had_failure and _is_finish(tool_name):+ # Any verdict queued behind the failure was written before the model saw it: a completed+ # one may be false, and a failed/terminated one carries a reason that predates the error.+ messages.append(+ {+ "role": "tool",+ "tool_call_id": tool_call_id,+ "name": tool_name,+ "content": (+ "skipped: a field in this batch failed before this verdict was reached; "+ "re-observe, then finish with a status that reflects the failure"+ ),+ }+ )+ continue+ if batch_had_failure and _may_submit(tool_name, args):+ messages.append(+ {+ "role": "tool",+ "tool_call_id": tool_call_id,+ "name": tool_name,+ "content": (+ "skipped: a field in this batch failed and this call may submit the form; "+ "fix the field, then re-queue it"+ ),+ }+ )+ continue# Once the action-step budget is spent, refuse a further page action — terminate, mirroring# the step engine's max-steps stop — but let perception/finish through, since the cap bounds# new action rounds, not the separate re-observe/finish turn the system prompt asks for.@@ -1184,6 +1282,11 @@ async def run_agent_tool_loop(await on_pre_action(tool_name, args)except Exception:LOG.warning("taskv3 on_pre_action callback failed", tool=tool_name, exc_info=True)+ # Sampled before dispatch so an error below can be checked for having moved the page even+ # when the tool set no flag (a raised exception carries none). Not billed to the tool's timing.+ probe_before: str | None = None+ if spec is not None and page_probe is not None:+ probe_before = await _sample_page_probe(page_probe)tool_started_at = time.monotonic()if spec is None:result = ToolResult.error(f"unknown_tool: {tool_name}")@@ -1250,6 +1353,11 @@ async def run_agent_tool_loop(if result.screenshots:pending_screenshots = list(result.screenshots)result_data = result.data or {}+ if tool_name == "look" and result_data.get("marks_renumbered"):+ # Every mark=N still queued in this batch was chosen before this look renumbered the+ # marks, so it now names an arbitrary element; a look refused before rebuilding its+ # manifest leaves the old marks (and their failed keys) live.+ marks_stale = Trueif progress is not None and observe_summary:invalid_fields = observe_summary.get("invalid_fields")if invalid_fields is not None and progress.observe(invalid_fields):@@ -1438,11 +1546,34 @@ async def run_agent_tool_loop(breakif result.status == "error":- # A failed call can leave the page in a state the rest of this batch was not planned- # against (e.g. a write after a failed navigate). Stop and skip the remaining calls so- # the model re-plans next turn from the error rather than acting on a stale assumption.- _append_skipped_tool_results(messages, tool_calls[idx + 1 :], "earlier tool call in this batch failed")- break+ # The rest of the batch is skipped only when the failed call moved the page: the tool's+ # own signal, or the probe sampled before dispatch (a missing reading counts as moved).+ poisoned = (+ tool_name == "navigate"+ or result.content == PAGE_UNAVAILABLE_ERROR+ or bool(+ result_data.get("page_transitioned")+ or result_data.get("page_state_changed")+ or result_data.get("navigation_dead_end")+ )+ )+ # Whether the page moved is independent of the tool's kind: a wait that timed out because+ # the site navigated poisons the batch just as a failed click would.+ if not poisoned and spec is not None and page_probe is not None:+ probe_after = await _sample_page_probe(page_probe)+ poisoned = probe_before is None or probe_after is None or probe_after != probe_before+ if poisoned:+ _append_skipped_tool_results(+ messages,+ tool_calls[idx + 1 :],+ "earlier tool call in this batch failed and changed the page — re-observe before "+ "re-queuing these",+ )+ break+ if spec is not None and (spec.billable or spec.recordable):+ batch_had_failure = True+ if call_selector is not None:+ failed_selectors.add(call_selector)# The batch settled on a dead page (an in-loop navigate hit a hard 404/410 and no later navigate# recovered): end the run as terminated deterministically, matching v1, rather than leaving thediff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 3beed2a6d..6ca31dffe 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -27,7 +27,13 @@ from PIL import Image, ImageDrawfrom skyvern.constants import BROWSER_DOWNLOADING_SUFFIXfrom skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stemfrom skyvern.forge.sdk.core.skyvern_context import URL_IN_TEXT, canonical_url, opaque_url_echo_window-from skyvern.forge.taskv3.loop import NAVIGATION_DEAD_END_STATUSES, ToolHandler, ToolResult, ToolSpec+from skyvern.forge.taskv3.loop import (+ NAVIGATION_DEAD_END_STATUSES,+ PAGE_UNAVAILABLE_ERROR,+ ToolHandler,+ ToolResult,+ ToolSpec,+)from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_actionif TYPE_CHECKING:@@ -41,8 +47,6 @@ LOG = structlog.get_logger()# tab/popup is followed on the next call instead of leaving the loop stuck on a stale page.PageProvider = Callable[[], Awaitable[Any]]-PAGE_UNAVAILABLE_ERROR = "browser page unavailable"-# Cap on the page URL observe() echoes. Callers that register a secret URL for exact-match redaction# must register this prefix too, or the truncated echo survives the scrub.OBSERVE_URL_MAX_CHARS = 300@@ -178,7 +182,9 @@ def _classify_commit("""Classify a value-must-change action from a before/after observable-state readback.Ranked fail-closed: a readable did-not-commit is reported whatever the target re-resolved to, because- only an error halts a batched turn (INV-1 guards the confident ok, not the refusal). A commit read off+ an error halts the rest of a batched turn only when it moved the page -- otherwise the field is+ reported unfilled and only its same-selector dependents and any later click or Enter are skipped+ (INV-1 guards the confident ok, not the refusal). A commit read offa target that re-resolved to n != 1 is `unverified` (INV-1); no readable committable state is`unverified` (INV-2). `committed_value` hands in a caller's own value-dimension truth in place of thegeneric any-field-changed rule.@@ -4447,7 +4453,8 @@ def build_browser_tools(if matches == 0:return ToolResult.error(f"{selector} no longer matches anything on the page — the page re-rendered since it was "- "observed. Re-observe and act on fresh selectors from the new observation."+ "observed. Re-observe and act on fresh selectors from the new observation.",+ data={"page_state_changed": True},)return ToolResult.error(f"{selector} matches {matches} elements, so it does not identify one control. Re-observe and "@@ -4679,7 +4686,8 @@ def build_browser_tools(return ToolResult.error(f"{selector} no longer exists on the page — element markers vanish when the ""page re-renders (a closed menu destroys its options). Re-observe and act on "- "fresh selectors from the new observation."+ "fresh selectors from the new observation.",+ data={"page_state_changed": True},)# The re-attach may have been a re-render that cloned the row, so the count is re-read.matches = await _marker_matches(page, selector)@@ -4689,7 +4697,8 @@ def build_browser_tools(return ToolResult.error(f"{selector} now matches {matches} elements — the page re-rendered and cloned the ""marked element, so the marker no longer identifies one control. Re-observe and act "- "on fresh selectors from the new observation."+ "on fresh selectors from the new observation.",+ data={"page_state_changed": True},)else:ambiguous = await _ambiguous_selector_error(page, selector)@@ -4808,7 +4817,8 @@ def build_browser_tools(await _clear_proxy_tags(page)return ToolResult.error(f"{selector} left the page before the click could land — it was replaced by a "- "re-render; re-observe and act on fresh selectors"+ "re-render; re-observe and act on fresh selectors",+ data={"page_state_changed": True},)base = f"clicked {selector} (hidden native control, toggled directly) — now at {await _url(page)}"else:@@ -4830,10 +4840,13 @@ def build_browser_tools(except Exception:gone = Falseif gone:+ # A same-document re-render: the URL and document nonce read unchanged, so only this+ # flag tells the loop the rest of the batch was planned against a stale page.return ToolResult.error(f"click on {selector} failed: the element no longer exists on the page — it was ""likely removed by a re-render (e.g. a menu closed and destroyed its options). "- f"Re-observe and act on fresh selectors. (original error: {type(e).__name__})"+ f"Re-observe and act on fresh selectors. (original error: {type(e).__name__})",+ data={"page_state_changed": True},)# Diagnosed only now, after the full actionability wait: a transient overlay (a toast,# a closing menu) deserves the whole 15s to clear on its own, not a probe-shortened one.@@ -5262,9 +5275,9 @@ def build_browser_tools(f"clicked suggestion {opt_txt!r} for {selector} but it did not commit — the field is ""NOT filled; re-observe and retry, do not proceed")- # DID_NOT_COMMIT: a suggestion surfaced but the field did not accept it — the field is NOT- # filled. Return an error so a batched turn halts here (the loop stops the rest of the batch- # on error) instead of proceeding — e.g. to a queued submit — on an uncommitted field.+ # DID_NOT_COMMIT: the field is NOT filled. The loop then skips any later click or Enter+ # in the same batch -- it may be an unvalidated submit, and no production submit guard+ # exists yet.return ToolResult.error(f"clicked suggestion {opt_txt!r} for {selector} but it did not commit — the field is NOT ""filled; re-observe and retry, do not proceed"@@ -5827,7 +5840,9 @@ def build_browser_tools(# bind the element fresh immediately before uploading (missing now means it vanished mid-download).el = await page.query_selector(selector)if el is None:- return ToolResult("error", f"no file input for selector {selector!r}", staged)+ return ToolResult(+ "error", f"no file input for selector {selector!r}", {**staged, "page_state_changed": True}+ )# Verify the upload took EFFECT, not just that set_input_files did not raise. Watch upload-like# network dispatches across the set_input_files + settle window (the window we already dwell in,# so this adds no latency); a genuine upload dispatches at least one, a silent no-op none.@@ -5961,6 +5976,8 @@ def build_browser_tools(except Exception:pass_look_manifest.clear()+ # From here on the old marks are gone, whatever happens next -- the loop keys on this.+ renumbered = {"marks_renumbered": True}for e in elements:n = int(e["n"])try:@@ -5983,10 +6000,11 @@ def build_browser_tools(annotated = await asyncio.get_running_loop().run_in_executor(None, _annotate_screenshot, png, kept, vw)except Exception as exc:LOG.warning("taskv3 look annotation failed", exc_info=True)- return ToolResult.error(f"look failed to render marks: {type(exc).__name__}: {exc}")+ return ToolResult.error(f"look failed to render marks: {type(exc).__name__}: {exc}", data=renumbered)if not kept:return ToolResult.ok("look: no interactive controls are visible in the viewport. Scroll or re-observe.",+ data=renumbered,screenshots=[annotated],)@@ -6009,7 +6027,7 @@ def build_browser_tools(if isinstance(data, dict) and data.get("truncated"):header += f" (only the first {_LOOK_MAX_MARKS} are marked; scroll for more.)"legend = header + "\n" + "\n".join(lines)- return ToolResult.ok(legend, screenshots=[annotated])+ return ToolResult.ok(legend, data=renumbered, screenshots=[annotated])async def _resolve_mark(page: Any, mark: int) -> tuple[str | None, ToolResult | None]:# Turn mark=N into a selector the existing click/type handlers act through. Resolution is the@@ -6034,7 +6052,8 @@ def build_browser_tools(if not connected:return None, ToolResult.error(f"mark {mark} no longer points to an element on the page — it moved or the page "- "re-rendered since look(). Call look() again and act on a fresh number."+ "re-rendered since look(). Call look() again and act on a fresh number.",+ data={"page_state_changed": True},)return f'[data-tv3-act="{mark}"]', Nonediff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 1483349eb..a906884d0 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -2537,12 +2537,14 @@ async def test_execute_task_v3_settle_completion_fenced_to_block_tasks(monkeypatch, outcome, task_block=block, data_extraction_goal=None, extracted_information_schema=None)assert loop_mock.await_args.kwargs["page_fingerprint"] is not None+ assert loop_mock.await_args.kwargs["page_probe"] is not None # batch-poisoning probe reaches the engineassert loop_mock.await_args.kwargs["max_settle_deferrals"] > 0_step, _task, bare_loop_mock, _post = await _run_execute_task_v3(monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None)assert bare_loop_mock.await_args.kwargs["page_fingerprint"] is not None+ assert bare_loop_mock.await_args.kwargs["page_probe"] is not Noneassert bare_loop_mock.await_args.kwargs["max_settle_deferrals"] == 0diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex c1f43f67b..74ee475e0 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -15,7 +15,7 @@ _CANDIDATE_METHODS = frozenset()_DISCOVERED_BROWSER_API_CALLS = {- "skyvern/forge/agent.py": Counter({"evaluate": 1, "new_page": 1}),+ "skyvern/forge/agent.py": Counter({"evaluate": 2, "new_page": 1}),"skyvern/forge/agent_functions.py": Counter({"close": 1, "scroll_into_view_if_needed": 1}),"skyvern/webeye/actions/handler.py": Counter({@@ -71,8 +71,9 @@ _DISCOVERED_BROWSER_API_CALLS = {}_EVALUATE_CALLERS = {- # Read-only DOM fingerprint sample for the v3 settle-before-complete check.- "skyvern/forge/agent.py": Counter({"_page_fingerprint": 1}),+ # Read-only DOM fingerprint sample for the v3 settle-before-complete check, and the per-document+ # nonce the v3 loop reads to tell whether a failed batched call navigated the page.+ "skyvern/forge/agent.py": Counter({"_page_fingerprint": 1, "_page_probe": 1}),"skyvern/webeye/actions/handler.py": Counter({"_blob_iframe_src_titles": 1,@@ -213,21 +214,21 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 156+ assert sum(sum(methods.values()) for methods in observed.values()) == 157handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 5- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 151+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 152def test_every_raw_evaluate_call_is_classified() -> None:observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}assert observed == _EVALUATE_CALLERS- assert sum(sum(callers.values()) for callers in observed.values()) == 25+ assert sum(sum(callers.values()) for callers in observed.values()) == 26def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 862979e8b..edfa19d89 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -461,6 +461,38 @@ async def test_engine_wires_failure_evidence_gate() -> None:assert outcome.reason == "still blocked, re-verified"+async def test_engine_forwards_the_page_probe_and_withholds_it_from_page_free_runs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Drop this kwarg anywhere on the way in and the loop classifies every unflagged error as+ # non-poisoning -- the fail-open state -- with every loop-level test still green.+ from skyvern.forge.taskv3 import engine as engine_mod+ from skyvern.forge.taskv3.loop import LoopOutcome++ captured: list[object] = []++ async def _capture(**kwargs: object) -> LoopOutcome:+ captured.append(kwargs.get("page_probe"))+ return LoopOutcome(status="completed", reason="ok")++ async def probe() -> str | None:+ return "doc-1"++ monkeypatch.setattr(engine_mod, "run_agent_tool_loop", _capture)+ await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()), llm_caller=_ScriptedCaller([]), goal="x", page_probe=probe+ )+ await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(_FakePage()),+ llm_caller=_ScriptedCaller([]),+ goal="x",+ page_probe=probe,+ page_free=True,+ )+ assert captured == [probe, None]++@pytest.mark.asyncioasync def test_engine_wires_the_pending_gate_and_withholds_it_from_page_free_runs(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex cd303fc38..9c3568bf7 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -24,6 +24,7 @@ from skyvern.exceptions import SkyvernContextWindowExceededErrorfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTaskfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.taskv3 import loop as loop_modulefrom skyvern.forge.taskv3.loop import (ACTION_LOOP_REASON_PREFIX,FAILURE_EVIDENCE_MIN_TOOL_CALLS,@@ -31,6 +32,7 @@ from skyvern.forge.taskv3.loop import (NAV_DEAD_END_REASON_PREFIX,NAVIGATION_DEAD_END_STATUSES,NO_TOOL_CALL_NUDGE,+ PAGE_UNAVAILABLE_ERROR,PERCEPTION_STALL_NUDGE_AFTER,PERCEPTION_STALL_REASON_PREFIX,PERCEPTION_STALL_SHADOW_EVENT,@@ -107,14 +109,44 @@ class _ScriptedCaller:return {"choices": [{"message": message}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}-def _recording_tool(name: str, sink: list[tuple[str, dict[str, Any]]], *, raises: bool = False) -> ToolSpec:+def _recording_tool(+ name: str, sink: list[tuple[str, dict[str, Any]]], *, raises: bool = False, billable: bool = False+) -> ToolSpec:async def handler(args: dict[str, Any]) -> ToolResult:sink.append((name, args))if raises:raise RuntimeError("boom")return ToolResult.ok(f"{name} done")- return ToolSpec(name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler)+ return ToolSpec(+ name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler, billable=billable+ )+++def _erroring_tool(+ name: str,+ sink: list[tuple[str, dict[str, Any]]],+ *,+ error_data: dict[str, Any] | None = None,+ billable: bool = False,+ recordable: bool = False,+) -> ToolSpec:+ """Like ``_recording_tool`` but returns ``ToolResult.error(...)`` directly (optionally carrying a+ ``data`` payload, e.g. ``page_transitioned``) instead of raising -- ``raises=True`` on+ ``_recording_tool`` only ever produces a bare ``tool_error: RuntimeError: boom`` with no data."""++ async def handler(args: dict[str, Any]) -> ToolResult:+ sink.append((name, args))+ return ToolResult.error(f"{name} failed", data=error_data)++ return ToolSpec(+ name=name,+ description=name,+ parameters={"type": "object", "properties": {}},+ handler=handler,+ billable=billable,+ recordable=recordable,+ )async def _run(script: list[list[tuple[str, dict[str, Any]]]], tools: list[ToolSpec], **kwargs: Any):@@ -167,13 +199,18 @@ async def test_perception_is_on_demand_never_injected() -> None:assert len(click_calls) == 2-def _look_tool(sink: list[tuple[str, dict[str, Any]]], *, image: bytes = b"\x89PNG-fake") -> ToolSpec:+def _look_tool(+ sink: list[tuple[str, dict[str, Any]]], *, image: bytes = b"\x89PNG-fake", fail_before_renumbering: bool = False+) -> ToolSpec:"""A `look`-shaped tool: returns a text legend AND a transient screenshot the loop must show the- model on the next call only (never persisted to the transcript)."""+ model on the next call only (never persisted to the transcript). Like the real tool it reports+ `marks_renumbered` once the manifest was rebuilt; a failure before that point reports nothing."""async def handler(args: dict[str, Any]) -> ToolResult:sink.append(("look", args))- return ToolResult.ok("[1] button 'Next'", screenshots=[image])+ if fail_before_renumbering:+ return ToolResult.error("look budget reached")+ return ToolResult.ok("[1] button 'Next'", data={"marks_renumbered": True}, screenshots=[image])return ToolSpec(name="look",@@ -678,14 +715,15 @@ async def test_tool_call_cap_stops_mid_batch() -> None:@pytest.mark.asyncioasync def test_tool_error_stops_batch_and_skips_remaining() -> None:- # A failed call mid-batch must stop the rest of the batch (so a later write can't run against a- # page a failed earlier call left in a bad state) and answer the skipped calls, so the next turn- # sees a valid transcript and re-plans from the error.+ # A failed call whose own result signals a page transition must still stop the rest of the batch+ # (so a later write can't run against a page the failed call left in a different state) and+ # answer the skipped calls, so the next turn sees a valid transcript and re-plans from the error.click_calls: list[tuple[str, dict[str, Any]]] = []+ boom_calls: list[tuple[str, dict[str, Any]]] = []type_calls: list[tuple[str, dict[str, Any]]] = []tools = [_recording_tool("click", click_calls),- _recording_tool("boom", [], raises=True),+ _erroring_tool("boom", boom_calls, error_data={"page_transitioned": True}),_recording_tool("type", type_calls),make_finish_tool(),]@@ -700,7 +738,634 @@ async def test_tool_error_stops_batch_and_skips_remaining() -> None:assert len(type_calls) == 0 # the call after the error was skipped, not executedturn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)- assert any("tool_error: RuntimeError" in m["content"] for m in turn1_tool_msgs)+ assert any(m.get("name") == "boom" and "boom failed" in m["content"] for m in turn1_tool_msgs)+++async def test_non_mutating_tool_error_lets_independent_batch_calls_run() -> None:+ # A `type` failure that leaves the page unchanged (no page-transition data, probe reads+ # unchanged) must not block unrelated select_option calls later in the same batch -- only a+ # failure that may have left the page in an unplanned-for state should stop the batch. `type`+ # and `select_option` are marked billable=True to match production (tools.py) so this exercises+ # the real probe-gated branch, not a fixture shortcut that skips it.+ click_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []+ select_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ tools = [+ _recording_tool("click", click_calls),+ _recording_tool("type", type_calls, raises=True, billable=True),+ _recording_tool("select_option", select_calls, billable=True),+ make_finish_tool(),+ ]+ script = [+ [+ ("click", {"selector": "#ok"}),+ ("type", {"selector": "#name"}),+ ("select_option", {"selector": "#a"}),+ ("select_option", {"selector": "#b"}),+ ("select_option", {"selector": "#c"}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(select_calls) == 3 # all three ran despite the earlier, non-page-mutating error+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "tool_error" in m["content"] for m in turn1_tool_msgs)+ assert not any("skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_failed_call_skips_only_same_selector_dependents() -> None:+ # A failed `type` on "#q" should skip a later call that targets the SAME selector (it depends on+ # the failed call having succeeded) but must not skip a call against an unrelated selector. All+ # three tools are billable=True to match production (tools.py), with a constant probe so the+ # probe-gated branch runs and reads unchanged.+ type_calls: list[tuple[str, dict[str, Any]]] = []+ press_calls: list[tuple[str, dict[str, Any]]] = []+ select_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ tools = [+ _recording_tool("type", type_calls, raises=True, billable=True),+ _recording_tool("press_key", press_calls, billable=True),+ _recording_tool("select_option", select_calls, billable=True),+ make_finish_tool(),+ ]+ script = [+ [+ ("type", {"selector": "#q"}),+ ("press_key", {"selector": "#q"}),+ ("select_option", {"selector": "#z"}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(press_calls) == 0 # depends on the failed #q type, must not run+ assert len(select_calls) == 1 # unrelated selector, must run+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(+ m.get("name") == "press_key" and "skipped" in m["content"] and "#q" in m["content"] for m in turn1_tool_msgs+ )+++async def test_failed_call_skips_only_same_mark_dependents() -> None:+ # A failed `type(mark=3)` should skip a later call on the SAME mark, mirroring the selector case,+ # since act-by-mark calls carry no top-level "selector" arg for _call_selector to key on. The+ # dependents are selects, not clicks: a click after any batch failure is deferred as a possible submit.+ type_calls: list[tuple[str, dict[str, Any]]] = []+ select_calls: list[tuple[str, dict[str, Any]]] = []+ tools = [+ _recording_tool("type", type_calls, raises=True, billable=True),+ _recording_tool("select_option", select_calls),+ make_finish_tool(),+ ]+ script = [+ [+ ("type", {"mark": 3}),+ ("select_option", {"mark": 3}),+ ("select_option", {"mark": 4}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools)++ assert outcome.status == "completed"+ assert len(select_calls) == 1 # only mark=4 dispatched; mark=3 depends on the failed type+ assert select_calls[0][1]["mark"] == 4+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "select_option" and "skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_mid_batch_look_defers_every_later_mark_call() -> None:+ # look() renumbers marks on every call, so a mark=3 queued behind a mid-batch look was chosen from+ # the OLD screenshot and now names an arbitrary element: it is deferred, not dispatched.+ type_calls: list[tuple[str, dict[str, Any]]] = []+ look_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ async def type_handler(args: dict[str, Any]) -> ToolResult:+ type_calls.append(("type", args))+ if len(type_calls) == 1:+ return ToolResult.error("type failed")+ return ToolResult.ok("type done")++ tools = [+ ToolSpec(+ name="type",+ description="type",+ parameters={"type": "object", "properties": {}},+ handler=type_handler,+ billable=True,+ ),+ _look_tool(look_calls),+ make_finish_tool(),+ ]+ script = [+ [+ ("type", {"mark": 3}),+ ("look", {}),+ ("type", {"mark": 3}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(look_calls) == 1+ assert len(type_calls) == 1 # the second mark=3 is deferred: its number predates the renumbering+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "type"]+ assert any("renumbered" in m["content"] for m in turn1_tool_msgs)+++async def test_non_page_action_failure_does_not_mark_its_selector_or_arm_the_batch() -> None:+ # A timed-out wait on #x mutates nothing: the later click on #x re-resolves the element itself, so+ # it must dispatch rather than be skipped as a dependent, and no submit deferral is armed.+ wait_calls: list[tuple[str, dict[str, Any]]] = []+ click_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ async def wait_handler(args: dict[str, Any]) -> ToolResult:+ wait_calls.append(("wait", args))+ return ToolResult.error("wait timed out")++ tools = [+ ToolSpec(+ name="wait", description="wait", parameters={"type": "object", "properties": {}}, handler=wait_handler+ ),+ _recording_tool("click", click_calls),+ make_finish_tool(),+ ]+ script = [+ [("wait", {"selector": "#x"}), ("click", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "done"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(wait_calls) == 1+ assert len(click_calls) == 1+++async def test_non_billable_failure_that_moved_the_page_still_stops_the_batch() -> None:+ # A wait that times out BECAUSE the site navigated is not a field failure, but the page moved: the+ # probe runs around every known tool, so the rest of the batch (planned for the old page) is skipped.+ readings = iter(["doc-1", "doc-2"])+ click_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return next(readings, "doc-2")++ async def wait_handler(args: dict[str, Any]) -> ToolResult:+ return ToolResult.error("wait timed out")++ tools = [+ ToolSpec(+ name="wait", description="wait", parameters={"type": "object", "properties": {}}, handler=wait_handler+ ),+ _recording_tool("click", click_calls),+ make_finish_tool(),+ ]+ script = [+ [("wait", {"selector": "#x"}), ("click", {"selector": "#next"})],+ [("finish", {"status": "terminated", "reason": "gave up"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "terminated"+ assert click_calls == []+ click_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "click"]+ assert any("changed the page" in m["content"] for m in click_msgs)+++async def test_hung_page_probe_is_bounded_and_reads_as_poisoned(monkeypatch: pytest.MonkeyPatch) -> None:+ # A renderer that never answers the probe must not stall the loop past its deadline: the sample is+ # bounded, and a missing reading is treated as poisoned (the batch stops), never as unchanged.+ monkeypatch.setattr(loop_module, "_PAGE_PROBE_TIMEOUT_SECONDS", 0.01)+ type_calls: list[tuple[str, dict[str, Any]]] = []++ async def hung_probe() -> str | None:+ await asyncio.Event().wait()+ return None++ async def type_handler(args: dict[str, Any]) -> ToolResult:+ type_calls.append(("type", args))+ if args.get("selector") == "#q":+ return ToolResult.error("type failed")+ return ToolResult.ok("type done")++ tools = [+ ToolSpec(+ name="type",+ description="type",+ parameters={"type": "object", "properties": {}},+ handler=type_handler,+ billable=True,+ ),+ make_finish_tool(),+ ]+ script = [+ [("type", {"selector": "#q"}), ("type", {"selector": "#zip"})],+ [("finish", {"status": "terminated", "reason": "gave up"})],+ ]+ outcome, _ = await asyncio.wait_for(_run(script, tools, page_probe=hung_probe), timeout=2)++ assert outcome.status == "terminated"+ assert [call_args.get("selector") for _, call_args in type_calls] == ["#q"] # batch stopped: reading missing+++async def test_look_that_fails_before_renumbering_keeps_mark_dependents() -> None:+ # A look() refused on budget (or failing to capture/enumerate) leaves the old manifest live, so a+ # mark=3 call after it still names the element whose earlier call failed and must stay skipped.+ type_calls: list[tuple[str, dict[str, Any]]] = []+ look_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ async def type_handler(args: dict[str, Any]) -> ToolResult:+ type_calls.append(("type", args))+ return ToolResult.error("type failed")++ tools = [+ ToolSpec(+ name="type",+ description="type",+ parameters={"type": "object", "properties": {}},+ handler=type_handler,+ billable=True,+ ),+ _look_tool(look_calls, fail_before_renumbering=True),+ make_finish_tool(),+ ]+ script = [+ [+ ("type", {"mark": 3}),+ ("look", {}),+ ("type", {"mark": 3}),+ ],+ [("finish", {"status": "terminated", "reason": "gave up"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "terminated"+ assert len(look_calls) == 1+ assert len(type_calls) == 1 # the second mark=3 call is still a dependent of the failed one+ type_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "type"]+ assert any("skipped" in m["content"] for m in type_msgs)+++ ("finish_status", "expected_turns", "expected_status"),+ [+ ("completed", 2, "completed"),+ ("terminated", 2, "terminated"),+ ("failed", 2, "failed"),+ ],+)+async def test_any_finish_deferred_after_batch_failure(+ finish_status: str, expected_turns: int, expected_status: str+) -> None:+ type_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ async def type_handler(args: dict[str, Any]) -> ToolResult:+ type_calls.append(("type", args))+ return ToolResult.error("type failed")++ tools = [+ ToolSpec(+ name="type",+ description="type",+ parameters={"type": "object", "properties": {}},+ handler=type_handler,+ billable=True,+ ),+ make_finish_tool(),+ ]+ script = [+ [("type", {"selector": "#q"}), ("finish", {"status": finish_status, "reason": "done"})],+ [("finish", {"status": finish_status, "reason": "done after re-checking"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ # Every verdict was written before the model saw the failure, so each is deferred one turn: a+ # completed one may be false, and a failed/terminated one carries a reason that predates the error.+ assert outcome.status == expected_status+ assert outcome.turns == expected_turns+ finish_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]+ assert any("skipped" in m["content"] for m in finish_msgs)+++async def test_cross_selector_dependent_call_still_dispatches_and_fails_on_its_own() -> None:+ # A call against a DIFFERENT selector than the failed one is not skipped by the same-selector+ # rule -- it dispatches and, if it truly depends on the failed call's DOM effect, fails on its+ # own terms rather than being wrong-committed as "skipped".+ calls: list[tuple[str, dict[str, Any]]] = []++ async def handler(args: dict[str, Any]) -> ToolResult:+ calls.append(("select_option", args))+ return ToolResult.error(f"no element for selector {args['selector']!r}")++ tools = [+ ToolSpec(+ name="select_option", description="s", parameters={"type": "object", "properties": {}}, handler=handler+ ),+ make_finish_tool(),+ ]+ script = [+ [("select_option", {"selector": "#a"}), ("select_option", {"selector": "#b"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools)++ assert outcome.status == "completed"+ assert len(calls) == 2 # #b was dispatched -- its selector differs from #a's, so it is not skipped+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "select_option"]+ assert all("skipped" not in m["content"] for m in turn1_tool_msgs)+ assert any("no element for selector '#b'" in m["content"] for m in turn1_tool_msgs)+++ "submit_call",+ [+ ("press_key", {"key": "Enter"}),+ ("press_key", {"key": "Control+Enter"}),+ ("press_key", {"selector": "#submit", "key": "Space"}),+ ("press_key", {"selector": "#submit", "key": " "}),+ ("type", {"selector": "#other", "press_enter": True}),+ ],+ ids=[+ "press_key_enter",+ "press_key_control_enter",+ "press_key_space",+ "press_key_literal_space",+ "type_press_enter",+ ],+)+async def test_click_and_enter_submit_skipped_after_batch_failure_but_other_fields_run(+ submit_call: tuple[str, dict[str, Any]],+) -> None:+ # After a page-action failure in the batch, the loop cannot classify a click -- it may be the+ # form's Submit -- so ANY later click is skipped alongside the Enter-shaped submit shapes. Other+ # field-filling tools (select_combobox, type, file_upload) on OTHER selectors are not submit-shaped+ # and still run.+ submit_name, submit_args = submit_call+ type_calls: list[tuple[str, dict[str, Any]]] = []+ click_calls: list[tuple[str, dict[str, Any]]] = []+ press_calls: list[tuple[str, dict[str, Any]]] = []+ combobox_calls: list[tuple[str, dict[str, Any]]] = []+ upload_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "doc-1"++ async def type_handler(args: dict[str, Any]) -> ToolResult:+ type_calls.append(("type", args))+ if args.get("selector") == "#q":+ return ToolResult.error("type failed")+ return ToolResult.ok("type done")++ tools = [+ ToolSpec(+ name="type",+ description="type",+ parameters={"type": "object", "properties": {}},+ handler=type_handler,+ billable=True,+ ),+ _recording_tool("click", click_calls),+ _recording_tool("press_key", press_calls),+ _recording_tool("select_combobox", combobox_calls),+ _recording_tool("file_upload", upload_calls),+ make_finish_tool(),+ ]+ script = [+ [+ ("type", {"selector": "#q"}),+ ("click", {"selector": "#agree"}),+ (submit_name, submit_args),+ ("select_combobox", {"selector": "#city"}),+ ("type", {"selector": "#zip"}),+ ("file_upload", {"selector": "#resume"}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(click_calls) == 0 # a click cannot be classified as safe, so it's skipped too+ if submit_name == "press_key":+ assert len(press_calls) == 0 # Enter-shaped submit skipped after the batch failure+ else:+ # The submit-shaped `type` call is skipped before dispatch -- only the earlier, failed "#q"+ # call and the later "#zip" call reach the handler and land in the sink.+ assert not any(call_args.get("selector") == "#other" for _, call_args in type_calls)+ assert len(combobox_calls) == 1 # unrelated field, not submit-shaped, still runs+ assert any(call_args.get("selector") == "#zip" for _, call_args in type_calls) # unrelated type still runs+ assert len(upload_calls) == 1 # unrelated field, not submit-shaped, still runs+ assert outcome.tool_calls == 5 # four dispatched calls plus finish: the two skipped calls cost no budget++ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "click" and "skipped" in m["content"] for m in turn1_tool_msgs)+ assert any(m.get("name") == submit_name and "skipped" in m["content"] for m in turn1_tool_msgs)+++ ("data_flag", "error_value", "expected_status"),+ [+ ("page_transitioned", True, "completed"),+ ("page_state_changed", True, "completed"),+ # navigation_dead_end additionally classifies the run as terminated once the batch settles --+ # that's a separate mechanism from the batch-stop this test targets, so it gets its own expected+ # final status rather than "completed".+ ("navigation_dead_end", 404, "terminated"),+ ],+)+async def test_page_changing_tool_error_still_stops_batch(+ data_flag: str, error_value: Any, expected_status: str+) -> None:+ # An error result that itself signals the page moved (page_transitioned, page_state_changed, or+ # navigation_dead_end in .data) must still stop the rest of the batch even though the call+ # "failed" -- the page moved out from under any planned follow-up regardless of the reported status.+ click_calls: list[tuple[str, dict[str, Any]]] = []+ nav_click_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []+ tools = [+ _recording_tool("click", click_calls),+ _erroring_tool("nav_click", nav_click_calls, error_data={data_flag: error_value}),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [+ ("click", {"selector": "#ok"}),+ ("nav_click", {"selector": "#nav"}),+ ("type", {"selector": "#x"}),+ ],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools)++ assert outcome.status == expected_status+ assert len(type_calls) == 0 # the batch stopped: the page moved under the failed call+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_navigate_tool_error_still_stops_batch() -> None:+ # A failed `navigate` carries no explicit page_transitioned/page_state_changed data, but+ # navigation is inherently page-mutating -- an errored navigate must still stop the batch.+ navigate_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []+ tools = [+ _recording_tool("navigate", navigate_calls, raises=True),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [("navigate", {"url": "https://example.com"}), ("type", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools)++ assert outcome.status == "completed"+ assert len(type_calls) == 0+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_page_unavailable_tool_error_still_stops_batch() -> None:+ # The page itself is gone: inherently poisoning regardless of tool name or data.+ async def handler(args: dict[str, Any]) -> ToolResult:+ return ToolResult.error(PAGE_UNAVAILABLE_ERROR)++ type_calls: list[tuple[str, dict[str, Any]]] = []+ tools = [+ ToolSpec(name="click", description="c", parameters={"type": "object", "properties": {}}, handler=handler),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools)++ assert outcome.status == "completed"+ assert len(type_calls) == 0+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_page_probe_change_across_failed_call_stops_batch() -> None:+ # A billable tool's error carries no data flag, but the page_probe sampled before and after the+ # dispatch shows the page changed underneath it -- still poisoning.+ click_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []+ probe_calls = {"n": 0}++ async def probe() -> str | None:+ probe_calls["n"] += 1+ return "A" if probe_calls["n"] == 1 else "B"++ tools = [+ _erroring_tool("click", click_calls, billable=True),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(type_calls) == 0 # the probe changed across the failed call -- batch stopped+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)+++async def test_page_probe_unchanged_across_failed_call_continues_batch() -> None:+ # Same shape as above, but the probe reads the same value before and after the failed call --+ # nothing in this error signals a page change, so the batch continues.+ click_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []++ async def probe() -> str | None:+ return "same"++ tools = [+ _erroring_tool("click", click_calls, billable=True),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [("click", {"selector": "#ok"}), ("type", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(type_calls) == 1 # the probe read unchanged across the failed call -- batch continues+++async def test_recordable_non_billable_tool_error_with_probe_change_stops_batch() -> None:+ # solve_captcha is recordable but not billable -- the probe must still be sampled around it, or a+ # failed solve that moved the page can never poison the batch.+ click_calls: list[tuple[str, dict[str, Any]]] = []+ type_calls: list[tuple[str, dict[str, Any]]] = []+ probe_calls = {"n": 0}++ async def probe() -> str | None:+ probe_calls["n"] += 1+ return "doc-1" if probe_calls["n"] == 1 else "doc-2"++ tools = [+ _erroring_tool("solve_captcha", click_calls, recordable=True),+ _recording_tool("type", type_calls),+ make_finish_tool(),+ ]+ script = [+ [("solve_captcha", {}), ("type", {"selector": "#x"})],+ [("finish", {"status": "completed", "reason": "recovered"})],+ ]+ outcome, _ = await _run(script, tools, page_probe=probe)++ assert outcome.status == "completed"+ assert len(type_calls) == 0 # the probe changed across the failed recordable call -- batch stopped+ turn1_tool_msgs = [m for m in outcome.messages if m.get("role") == "tool"]+ assert any(m.get("name") == "type" and "skipped" in m["content"] for m in turn1_tool_msgs)@pytest.mark.asyncio@@ -1231,7 +1896,9 @@ async def test_every_executed_tool_call_emits_one_timing_record() -> None:# A null selector is the case that matters: the tools fall back to scanning the whole page,# so it must read as absent even though the key is present.[("observe", {"selector": "sel"}), ("click", {"selector": None})],- [("boom", {}), ("click", {})], # boom fails, so the trailing click is skipped, not executed+ # boom fails on "#s"; the trailing click on the SAME selector depends on it and is skipped+ # before dispatch (SKY-15143: a non-page-mutating error no longer halts the whole batch).+ [("boom", {"selector": "#s"}), ("click", {"selector": "#s"})],[("nope", {})], # unknown tool[("finish", {"status": "completed", "reason": "ok"})],]@@ -1248,7 +1915,7 @@ async def test_every_executed_tool_call_emits_one_timing_record() -> None:assert [entry["tool_status"] for entry in records] == ["ok", "ok", "error", "error", "ok"]assert [entry["batch_size"] for entry in records] == [2, 2, 2, 1, 1]assert [entry["batch_index"] for entry in records] == [0, 1, 0, 0, 0]- assert [entry["selector_present"] for entry in records] == [True, False, False, False, False]+ assert [entry["selector_present"] for entry in records] == [True, False, True, False, False]assert [entry["billable"] for entry in records] == [False, False, True, False, False]assert [entry["turn"] for entry in records] == [1, 1, 2, 3, 4]diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex d2fb597d5..5c13d7d56 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -5163,6 +5163,7 @@ async def test_click_on_a_marker_the_page_cloned_never_silently_lands_on_the_cloassert clicked == [], f"failed loud yet still dispatched a click on {clicked}"assert "e-observe" in r.content, r.contentassert selector in r.content+ assert r.data == {"page_state_changed": True} # a cloning re-render must poison the batch@_skip_no_browser@@ -5212,6 +5213,7 @@ async def test_click_on_a_marker_the_page_destroyed_still_fails_loud() -> None:assert r.status == "error"assert "no longer exists" in r.content and "e-observe" in r.content+ assert r.data == {"page_state_changed": True} # a same-document re-render must poison the batchassert await page.evaluate("() => window.__clicked") == []