Skip to content

Automate browser based workflows with AI

Stop charging Task V3 action steps for calls refused before acting on the page (#8688)

pedrohsdb6a88ba618305
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index d99830a6d..e5e8864df 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -201,6 +201,12 @@ class ToolResult:
# OTHER one drops it silently; the classmethods below cannot express that pairing (neither
# accepts the other's kwarg) and the raw constructor is the only route that could.
ok_class: ToolOkClass | None = None
+ # The requested action was refused and never dispatched (a ToolRefusal). A field, not a raise, so
+ # every wrapper around the handler still post-processes the call.
+ refused: bool = False
+ # A refused call that still dispatched input to the page first, to release a previous field's
+ # suggestion list. It is charged and the page may have changed, though the requested action never ran.
+ touched_page: bool = False
@classmethod
def ok(
@@ -224,12 +230,21 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]
class ToolRefusal(Exception):
- """Raised below a handler's return path to refuse the call; the loop sends the message alone as
- the tool's error, so it must be written for the model and carry nothing it must not see."""
+ """Refuse the call before it acts on the page (no input dispatched; a marker attribute written to
+ address an element is not an action). The loop sends the message alone as a refused error, which
+ does not spend an action step within the grace, so it must carry nothing the model must not see."""
- def __init__(self, message: str, *, error_class: ToolErrorClass) -> None:
+ def __init__(self, message: str, *, error_class: ToolErrorClass, data: dict[str, Any] | None = None) -> None:
super().__init__(message)
self.error_class: ToolErrorClass = error_class
+ self.data = data
+
+ @classmethod
+ def of(cls, result: ToolResult) -> ToolRefusal:
+ return cls(result.content, error_class=result.error_class or "other", data=result.data)
+
+ def as_result(self) -> ToolResult:
+ return ToolResult("error", str(self), self.data, error_class=self.error_class, refused=True)
# The tool-result `data` keys the target-name/target-kind capture ride on (written by the browser
@@ -677,6 +692,9 @@ ACTION_BUDGET_EXTENSION_EVIDENCE_WINDOW = 8
# Total growth stops at this multiple of the original cap. The wall clock — the one cap that is not
# a function of the action-step budget — is the outer runaway stop and is never re-derived.
ACTION_BUDGET_EXTENSION_MAX_FACTOR = 3
+# Refusals left uncharged in a row before a refusal is charged like any failed call. The repeat guard cannot
+# bound them (a stale address resets it), so this does.
+UNCHARGED_REFUSAL_GRACE = ACTION_LOOP_NUDGE_AFTER
# Facetable event names — both the grant and the refusal are queryable so the gate's decision
# precision is measurable on the canary; change only with the dashboards that read them.
ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"
@@ -2962,6 +2980,8 @@ class LoopState:
tool_seconds: float = 0.0
total_tokens: int = 0
action_steps: int = 0
+ # Refused billable calls left uncharged since the last charged, non-refused one.
+ uncharged_refusals: int = 0
budget_extended_notice: str | None = None
# Progress-gated budget extension (SKY-15264, SKY-15666): how many extensions have been granted,
# and the cap they are all sized and bounded against — captured before the first grant so growth
@@ -3604,7 +3624,10 @@ async def run_agent_tool_loop(
st.messages.append({"role": "user", "content": NO_TOOL_CALL_NUDGE})
continue
- turn_did_action = False
+ # Dispatched feeds the page-state stall detector; charged feeds the action-step budget and the
+ # rows' billable flag. They differ only by calls the tool refused before acting on the page.
+ turn_dispatched_billable = False
+ turn_charged = False
st.stall_nudges_due = []
st.refresh_nudge_due = False
st.budget_extended_notice = None
@@ -3629,7 +3652,7 @@ async def run_agent_tool_loop(
batch_page_change_reason: str | None = None
batch_fp_before: str | None = None
# Sample only when the batch can actually land a billable action -- the end-of-batch check
- # below gates on turn_did_action, so a finish-only or perception-only batch has no use for
+ # below gates on turn_dispatched_billable, so a finish-only or perception-only batch has no use for
# this baseline and shouldn't pay its round-trip.
batch_has_billable_call = any(
tool_by_name.get(tool_name) is not None and tool_by_name[tool_name].billable
@@ -3651,7 +3674,7 @@ async def run_agent_tool_loop(
# The page moved BETWEEN batches (a delayed render landing after the prior
# after-sample): the touches the old samples described are stale, and this batch's
# dispatch and extension decisions must not read them. Canonical-only — the
- # incumbent stall counters keep their end-of-batch turn_did_action gate.
+ # incumbent stall counters keep their end-of-batch turn_dispatched_billable gate.
st.canonical.progress(_ProgressEvidence.CROSS_BATCH_MOVEMENT)
batch_fp_after: str | None = None
if activity is not None:
@@ -4084,13 +4107,11 @@ async def run_agent_tool_loop(
result = ToolResult.error(f"unknown_tool: {tool_name}")
else:
if spec.billable:
- # A dispatched page action consumes a step even if it errors (it may mutate before
- # failing); billing below counts successes only.
- turn_did_action = True
+ turn_dispatched_billable = True
try:
result = await spec.handler(args)
except ToolRefusal as refusal:
- result = ToolResult.error(str(refusal), error_class=refusal.error_class)
+ result = refusal.as_result()
except Exception as exc:
LOG.warning(
"taskv3 tool handler raised",
@@ -4102,6 +4123,16 @@ async def run_agent_tool_loop(
result = ToolResult.error(f"tool_error: {type(exc).__name__}: {exc}", error_class=raised_class)
tool_duration_seconds = time.monotonic() - tool_started_at
st.tool_seconds += tool_duration_seconds
+ # A dispatched page action consumes a step even if it errors (it may mutate before failing),
+ # unless the tool refused it before acting on the page, within the grace. Billing counts
+ # successes only.
+ call_charged = spec is not None and spec.billable
+ if call_charged and result.status == "error" and result.refused and not result.touched_page:
+ call_charged = st.uncharged_refusals >= UNCHARGED_REFUSAL_GRACE
+ st.uncharged_refusals += 0 if call_charged else 1
+ elif call_charged:
+ st.uncharged_refusals = 0
+ turn_charged = turn_charged or call_charged
# Entry is recorded only when the tool succeeded (a stale-ref failure put nothing in the
# field), but the submit is recorded on DISPATCH whatever the verdict: the loop ran the
# click itself, so nothing here asks the page whether a submission completed. Entry first,
@@ -4133,6 +4164,10 @@ async def run_agent_tool_loop(
# vocabulary with Python exception names under one facet, and taskv3 emits on every
# erroring tool call so it would dominate the values.
cost_fields["tool_error_class"] = result.error_class or "other"
+ # Only on the rows it describes, like the fields below: a billable call that did not
+ # spend an action step because the tool refused it before acting on the page.
+ if spec is not None and spec.billable and not call_charged:
+ cost_fields["charged"] = False
# Only on the class they describe, so every other erroring row keeps exactly the
# fields it has today. Total over `covered` rows by construction: the helper that
# builds all three messages records before it returns any of them, so a covered row
@@ -4317,7 +4352,7 @@ async def run_agent_tool_loop(
status=result.status,
content=model_facing_content,
perception=spec.compactable,
- page_changing=spec.touches_page,
+ page_changing=spec.touches_page and (result.touched_page or not result.refused),
# The two ways a secret reaches the page: a credential placeholder typed by an
# entry tool, and a value a tool registered as secret (a delivered verification
# code) for the model to type next.
@@ -4397,8 +4432,9 @@ async def run_agent_tool_loop(
if spec is not None and spec.billable:
if st.progress is not None:
st.progress.on_billable()
- # Errored dispatches count too: a failed attempt consumed a step (see the action-step
- # accounting above) and a repeat-failing action is the same no-progress pathology.
+ # Errored and refused dispatches count too: a repeat-failing action is the same
+ # no-progress pathology. It does not bound refusals (a stale one resets it);
+ # UNCHARGED_REFUSAL_GRACE does.
repeat_count, first_turn, streak_fp, streak_moved = st.action_counts.get(
action_key, (0, st.turns, batch_fp_before, False)
)
@@ -4459,9 +4495,9 @@ async def run_agent_tool_loop(
elif spec.compactable and result.status == "ok":
activity.perceptions += 1
if spec is not None and (spec.billable or spec.recordable):
- # Dispatched page actions enter the round with their outcome: a failed billable round
+ # Dispatched page actions enter the round with their outcome: a failed charged round
# still consumed budget and must persist (else later blocks undercount the run
- # budget); recordable tools persist for artifact parity without billing/budget.
+ # budget); refused calls and recordable tools persist without claiming a budget unit.
round_outcome = result_data.get(ACTION_OUTCOME_DATA_KEY)
round_outcome = round_outcome if isinstance(round_outcome, dict) else None
round_actions.append(
@@ -4471,7 +4507,7 @@ async def run_agent_tool_loop(
result.status == "ok" and not _outcome_reports_failure(round_outcome),
result_data.get(TARGET_LABEL_DATA_KEY) or None,
result_data.get(TARGET_KIND_DATA_KEY) or None,
- spec.billable,
+ call_charged,
round_outcome,
result.content if result.status == "error" else None,
)
@@ -4649,7 +4685,12 @@ async def run_agent_tool_loop(
# leaves the rendered document byte-identical ticks the counter, whatever tools produced it.
# A missing sample is no evidence either way; any page-change flag or fingerprint movement
# re-baselines.
- if st.outcome is None and turn_did_action and page_fingerprint is not None and batch_fp_before is not None:
+ if (
+ st.outcome is None
+ and turn_dispatched_billable
+ and page_fingerprint is not None
+ and batch_fp_before is not None
+ ):
if st.page_state_prev_fp is not None and batch_fp_before != st.page_state_prev_fp:
# The page moved BETWEEN batches (a delayed render landing after the prior
# after-sample): the streak the old samples described is stale.
@@ -4783,7 +4824,7 @@ async def run_agent_tool_loop(
# turns (observe/get_html) don't consume the caller's step budget — the step engine bundles
# perception into each step, so counting v3's perception rounds against the same budget
# under-counts equivalent work.
- if turn_did_action:
+ if turn_charged:
st.action_steps += 1
# Hand the round's executed actions to the caller so it can persist per-action artifacts
# (screenshot, DB rows) — kept out of this transport-agnostic core, like should_cancel. A
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 8b89d234e..b52465092 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -15,6 +15,7 @@ from __future__ import annotations
import asyncio
import contextlib
import dataclasses
+import functools
import io
import json
import os
@@ -357,6 +358,8 @@ def _escape_tags_in_text(text: str) -> str:
_TV3_MARKER_SELECTOR_RE = re.compile(
r'^\[data-tv3(?:(?:-menu|-act)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$'
)
+# Any v3 marker inside a selector, including one observe composed a host-anchored selector around.
+_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg)?="')
# An opaque identifier (a uuid, or a run of 12+ hex digits) does not survive a model's copy: one
# transposed pair sends every later call to a selector that matches nothing. observe addresses its
# own elements by ref for that reason; this is what look()'s legend refuses to use as a label.
@@ -471,6 +474,20 @@ def _inert_target_error(selector: str) -> ToolResult:
)
+def _with_refusal_result(handler: ToolHandler) -> ToolHandler:
+ """Turn a raised ToolRefusal into its refused result, so the wrappers outside this one post-process
+ a refused call (download notices, frame work, typeahead release) like any other."""
+
+ @functools.wraps(handler)
+ async def wrapped(args: dict[str, Any]) -> ToolResult:
+ try:
+ return await handler(args)
+ except ToolRefusal as refusal:
+ return refusal.as_result()
+
+ return wrapped
+
+
def _with_selector_guard(handler: ToolHandler, diagnose_inert: InertTargetDiagnosis | None = None) -> ToolHandler:
"""Shared seam for selector tools: normalize a bare invalid `#id` before the handler resolves it, and
convert a residual invalid-selector crash into an actionable error instead of a batch-aborting raise."""
@@ -11211,29 +11228,36 @@ def build_browser_tools(
except Exception:
return 1 # count unavailable → do not block, mirroring _marker_matches' fail-open
- async def _ambiguous_selector_error(page: Any, selector: str) -> ToolResult | None:
+ async def _require_single_target(page: Any, selector: str) -> None:
+ """Raise ToolRefusal, before acting on the page, when a counted selector names no one element."""
# A host-anchored selector straddles a shadow boundary, which the per-root marker count
# cannot see through; the executor's own engine can, so it supplies the count. Playwright's
# actions are non-strict and would otherwise land on whichever match comes first.
if not (_is_host_anchored_selector(selector) or _TV3_MARKER_SELECTOR_RE.match(selector.strip())):
- return None
+ return
try:
matches = await page.locator(selector).count()
except Exception:
# Left open, as the marker count is: refusing here would block every action on a page
# whose engine hiccups, and the action's own actionability wait still applies.
LOG.warning("taskv3 selector count unavailable; acting unverified", selector=selector)
- return None
+ return
if matches == 1:
- return None
- if matches == 0:
- return ToolResult.error(
+ return
+ names_marker = bool(_TV3_MARKER_ANYWHERE_RE.search(selector))
+ if matches == 0 and names_marker:
+ raise ToolRefusal(
f"{selector} no longer matches anything on the page — the page re-rendered since it was "
"observed. Re-observe and act on fresh selectors from the new observation.",
data={"page_state_changed": True},
error_class="stale_selector",
)
- return ToolResult.error(
+ if matches == 0:
+ raise ToolRefusal(
+ f"{selector} matches nothing on the page. Re-observe and act on a ref from the observation.",
+ error_class="stale_selector",
+ )
+ raise ToolRefusal(
f"{selector} matches {matches} elements, so it does not identify one control. Re-observe and "
"act on a selector from the new observation, or narrow this one until it matches exactly one.",
error_class="ambiguous_selector",
@@ -11488,7 +11512,7 @@ def build_browser_tools(
try:
await page.wait_for_selector(selector, state="attached", timeout=1200)
except Exception:
- return ToolResult.error(
+ raise ToolRefusal(
f"{selector} no longer exists on the page — element markers vanish when the "
"page re-renders (a closed menu destroys its options). Re-observe and act on "
"fresh selectors from the new observation.",
@@ -11500,7 +11524,7 @@ def build_browser_tools(
if matches > 1:
# A clone of the marked element carries the same marker; the click would silently
# land on whichever comes first in document order, so refuse before dispatching it.
- return ToolResult.error(
+ raise ToolRefusal(
f"{selector} now matches {matches} elements — the page re-rendered and cloned the "
"marked element, so the marker no longer identifies one control. Re-observe and act "
"on fresh selectors from the new observation.",
@@ -11508,9 +11532,7 @@ def build_browser_tools(
error_class="ambiguous_selector",
)
else:
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
pre: dict[str, Any] | None = None
try:
pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))
@@ -11930,9 +11952,7 @@ def build_browser_tools(
if error is not None:
return error
selector = args["selector"]
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
await page.hover(selector, timeout=_ACTION_TIMEOUT_MS)
return ToolResult.ok(f"hovered {selector}")
@@ -12890,6 +12910,7 @@ def build_browser_tools(
await _resolve_mirrored_host_control(page, selector) if isinstance(selector, str) and selector else None
)
on_field: list[tuple[str, _LiveRowOffer]] = []
+ released_input = False
for field, offer in list(_live_row_offers.items()):
try:
relation = await offer.page.evaluate(
@@ -12961,11 +12982,14 @@ def build_browser_tools(
):
continue
# A page can replace the field during cleanup; writes and Escape must stay on the original node.
+ # released_input is set only once a dispatch returned, so a fill that raised first stays uncharged.
if current:
await offer.element.fill("", timeout=_ACTION_TIMEOUT_MS)
+ released_input = True
if await offer.element.evaluate(read_value) != "":
continue
dismissed = await _close_lingering_typeahead_list(offer.page, field, None, element=offer.element)
+ released_input = True
if dismissed is not None:
return ToolResult.error(
"The requested action did not run. Closing the previous field's still-open list with "
@@ -12981,6 +13005,9 @@ def build_browser_tools(
result = await handler(args)
finally:
_prefetched_page.clear()
+ if released_input and result.refused:
+ # Releasing the previous field already acted on the page, so this call is charged.
+ result = dataclasses.replace(result, touched_page=True)
if result.status == "ok" and tool_name in {"select_combobox", "type"}:
for field, offer in on_field:
if _live_row_offers.get(field) is offer:
@@ -13766,9 +13793,7 @@ def build_browser_tools(
selector = args.get("selector")
if not selector:
return ToolResult.error("type needs a selector, or mark=N from the last look().")
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
selector = await _resolve_mirrored_host_control(page, selector)
text = await _resolve_text(args.get("text", ""), operation="type", page=page, selector=selector)
press_enter = args.get("press_enter")
@@ -15212,9 +15237,7 @@ def build_browser_tools(
value = args.get("value")
label_list = _option_str_list(args.get("labels"))
value_list = _option_str_list(args.get("values"))
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
selector = await _resolve_mirrored_host_control(page, selector)
try:
probe = await _probe_evaluate(page, _SELECT_VISIBILITY_JS, selector, await _probe_arg(page, selector))
@@ -15342,9 +15365,7 @@ def build_browser_tools(
# The tab's URL, like click's: the re-ask's URL rule reads where a key press that may submit started.
url_before = await _url(_current_page())
if selector:
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
await page.press(selector, key)
else:
# Page-level, not realm-level: a Frame has no `keyboard`, and an unaddressed keypress goes
@@ -15360,9 +15381,7 @@ def build_browser_tools(
return error
selector = args.get("selector")
if selector:
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
el = await page.query_selector(selector)
if el:
await el.scroll_into_view_if_needed()
@@ -15555,9 +15574,7 @@ def build_browser_tools(
if error is not None:
return error
selector = args["selector"]
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
# Validate the selector before fetching: an invalid or missing selector fails here, before anything
# is staged into downloads_dir, so the selector guard's residual error can never leave a phantom
# upload for the download-signal wrapper to misread as a browser download.
@@ -15735,9 +15752,7 @@ def build_browser_tools(
if error is not None:
return error
selector = args["selector"]
- ambiguous = await _ambiguous_selector_error(page, selector)
- if ambiguous is not None:
- return ambiguous
+ await _require_single_target(page, selector)
selector = await _resolve_mirrored_host_control(page, selector)
offer = _live_row_offers.get(selector)
live_offer = None
@@ -16065,7 +16080,7 @@ def build_browser_tools(
# Its neighbour above ("either mark or selector, not both") deliberately keeps no
# class -- that is a schema error with no counterpart in the css cohort, so naming
# it would add a value to one side of the comparison this record exists to support.
- return ToolResult.error(
+ raise ToolRefusal(
f"mark must be an integer from the last look(), got {mark!r}.",
error_class="invalid_mark",
)
@@ -16081,7 +16096,7 @@ def build_browser_tools(
# rather than hiding exactly the failure cost this field exists to price.
selector, mark_error = await _resolve_mark(page, mark_int)
if mark_error is not None:
- return mark_error
+ raise ToolRefusal.of(mark_error)
# In place rather than into a copy: everything downstream reads this dict AFTER dispatch
# -- the persisted action's element_id, the submit watch, the repeat guard's key and the
# nudge's target -- and a copy leaves every one of them seeing only `mark`.
@@ -16438,7 +16453,7 @@ def build_browser_tools(
return error
realm, realm_error = await _realm_for_typed_selector(page, selector)
if realm_error is not None:
- return realm_error
+ raise ToolRefusal.of(realm_error)
# The routed page is handed on, because the realm decision was made ABOUT it: the
# provider is must_get_working_page and can switch to a newer tab, so letting preflight
# resolve again would run the selector on a popup that became valid during the frame
@@ -16468,7 +16483,7 @@ def build_browser_tools(
ref = int(match.group(1))
resolved, ref_error = await _resolve_ref(page, ref)
if ref_error is not None:
- return ref_error
+ raise ToolRefusal.of(ref_error)
_owner = (_observe_manifest.get(ref) or {}).get("owner")
# In place, for the same reason act-by-mark does it: the persisted action's element_id,
# the submit watch, the repeat guard's key and the nudge all read this dict AFTER dispatch.
@@ -16721,6 +16736,8 @@ def build_browser_tools(
# "not in the current set of marks" (a clean no-op), so click/type need no further change.
tools = [t for t in tools if t.name != "look"]
for _tool_spec in tools:
+ # Innermost, so every wrapper below sees a refused call as a result.
+ _tool_spec.handler = _with_refusal_result(_tool_spec.handler)
if _tool_spec.name in (
"click",
"hover",
@@ -16768,6 +16785,8 @@ def build_browser_tools(
# args["selector"], so the whole verified click/type path (uniqueness gate, commit-verify)
# runs on the act-by-mark selector unchanged.
_tool_spec.handler = _with_act_by_mark(_tool_spec.handler)
+ # Again outermost, for the refusals the mark and ref resolvers raise themselves.
+ _tool_spec.handler = _with_refusal_result(_tool_spec.handler)
if _tool_spec.name == "click":
_tool_spec.toggle_probe = _click_targets_toggle
_apply_download_signal(tools, downloads_dir)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index b0e176470..03bf77fef 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -4877,6 +4877,34 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(
]
+async def test_execute_task_v3_refused_round_persists_a_failed_row_without_a_budget_unit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The loop marks a call refused before it touched the page with billable=False and does not count
+ # it in action_steps. The pool counts distinct (task_id, step_order) pairs, so a refused round has
+ # to ride an index a charged round claims: 2 charged rounds here, so exactly 2 pairs.
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click", "click"])
+ refused = RoundAction("click", {"selector": "text=Open Items"}, False, billable=False, error="matches 2 elements")
+ rounds = [
+ [refused],
+ [RoundAction("click", {"selector": "#a"}, True, billable=True)],
+ [refused],
+ [RoundAction("click", {"selector": "#b"}, True, billable=True)],
+ ]
+ await _run_execute_task_v3(
+ monkeypatch, outcome, action_rounds=rounds, data_extraction_goal=None, extracted_information_schema=None
+ )
+ action_rows = agent_module.app.DATABASE.workflow_params.create_action.await_args_list[:-1]
+ stamped = [(c.kwargs["action"].status, c.kwargs["action"].step_order) for c in action_rows]
+ assert stamped == [
+ (ActionStatus.failed, 0),
+ (ActionStatus.completed, 0),
+ (ActionStatus.failed, 0),
+ (ActionStatus.completed, 1),
+ ]
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_failed_run_carries_failure_category(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_multi_field_totp.py b/tests/unit/test_multi_field_totp.py
index e218d2063..c2d0fd094 100644
--- a/tests/unit/test_multi_field_totp.py
+++ b/tests/unit/test_multi_field_totp.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import hashlib
+import inspect
import json
import shutil
import subprocess
@@ -867,8 +868,9 @@ async def test_multi_field_totp_browser_privacy_after_fill(monkeypatch: pytest.M
assert len(legend) == 6
assert [line.split("input ", 1)[1] for line in legend] == [repr("*" * len(value)) for value in values]
# The same sanitized labels are retained for subsequent mark actions.
+ raw_look = inspect.unwrap(look.handler)
look_state = dict(
- zip(look.handler.__code__.co_freevars, [cell.cell_contents for cell in look.handler.__closure__])
+ zip(raw_look.__code__.co_freevars, [cell.cell_contents for cell in raw_look.__closure__])
)
assert [look_state["_look_manifest"][i]["label"] for i in range(1, 7)] == [
"*" * len(value) for value in values
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 325490817..4c789f499 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import copy
+import dataclasses
import hashlib
import json
import random
@@ -62,6 +63,7 @@ from skyvern.forge.taskv3.loop import (
PERCEPTION_STALL_TERMINATE_AFTER,
PROGRESS_LEDGER_SHADOW_EVENT,
PROGRESS_LEDGER_WINDOW,
+ UNCHARGED_REFUSAL_GRACE,
VERDICT_URL_MAX_CHARS,
ActivityRecency,
LoopOutcome,
@@ -69,6 +71,7 @@ from skyvern.forge.taskv3.loop import (
SemanticCommitStats,
SubmitWatch,
ToolHandler,
+ ToolRefusal,
ToolResult,
ToolSpec,
_arms_failure_evidence,
@@ -2478,6 +2481,128 @@ async def test_action_step_budget_extension_truncated_to_workflow_run_ceiling()
assert extended and extended[0]["extension"] == 1
+def _click_then(first: ToolResult | ToolRefusal, sink: list[tuple[str, dict[str, Any]]]) -> ToolSpec:
+ # The first call answers `first` (returned, or raised when it is a refusal); every later one lands.
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ sink.append(("click", args))
+ if len(sink) == 1:
+ if isinstance(first, ToolRefusal):
+ raise first
+ return first
+ return ToolResult.ok("clicked")
+
+ return ToolSpec(
+ name="click",
+ description="click",
+ parameters={"type": "object", "properties": {}},
+ handler=handler,
+ billable=True,
+ )
+
+
+async def _atomic_block_run(
+ first: ToolResult | ToolRefusal,
+) -> tuple[LoopOutcome, int, list[Any], list[list[RoundAction]]]:
+ # An atomic Action block: one action step and no extension (the ceiling pins it to the cap).
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ rounds: list[list[RoundAction]] = []
+
+ async def _on_round(round_actions: list[RoundAction], _text: str | None) -> None:
+ rounds.append(list(round_actions))
+
+ script = [
+ [("click", {"selector": "text=Open Items"})],
+ [("click", {"selector": "ref=3"})],
+ [("finish", {"status": "completed", "reason": "done"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [_click_then(first, clicks), make_finish_tool()],
+ max_action_steps=1,
+ max_action_steps_ceiling=1,
+ on_action_round=_on_round,
+ )
+ assert clicks[0] == ("click", {"selector": "text=Open Items"})
+ records = [e for e in logs if e["event"] == "taskv3 tool call finished" and e["tool"] == "click"]
+ return outcome, len(clicks), records, rounds
+
+
+async def test_a_refusal_before_the_page_is_touched_does_not_spend_the_atomic_blocks_one_round() -> None:
+ refusal = ToolRefusal("text=Open Items matches 2 elements", error_class="ambiguous_selector")
+ outcome, dispatched, records, rounds = await _atomic_block_run(refusal)
+
+ assert dispatched == 2, "the real click after the refusal was skipped as over budget"
+ assert outcome.status == "completed"
+ assert outcome.cap_trip is None
+ assert outcome.action_steps == 1
+ # The workflow-run pool counts rounds off RoundAction.billable: it must agree with action_steps.
+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, False)], [(True, True)]]
+ assert records[0]["tool_error_class"] == "ambiguous_selector"
+ assert records[0]["charged"] is False
+ assert "charged" not in records[1]
+
+
+async def test_an_error_after_the_act_still_spends_the_atomic_blocks_one_round() -> None:
+ # Shaped like a click whose page.click landed and re-rendered the target away: the tool may have
+ # mutated the page, so nothing but a declared refusal goes uncharged.
+ post_act = ToolResult.error(
+ "click on text=Open Items failed: the element no longer exists on the page",
+ data={"page_state_changed": True},
+ error_class="stale_selector",
+ )
+ outcome, dispatched, records, rounds = await _atomic_block_run(post_act)
+
+ assert dispatched == 1
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+ assert outcome.action_steps == 1
+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, True)]]
+ assert "charged" not in records[0]
+
+
+async def test_a_refusal_that_touched_the_page_first_spends_the_atomic_blocks_one_round() -> None:
+ # The tool released another field's suggestion list before refusing: input reached the page.
+ touched = dataclasses.replace(
+ ToolRefusal("#sign-in .button matches 2 elements", error_class="ambiguous_selector").as_result(),
+ touched_page=True,
+ )
+ outcome, dispatched, _records, rounds = await _atomic_block_run(touched)
+
+ assert dispatched == 1
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, True)]]
+
+
+async def test_an_atomic_block_that_is_refused_forever_ends_after_the_grace_with_its_step_charged() -> None:
+ # Each refusal names a new address and reports a page change, so the repeat guard never binds;
+ # the grace is the only thing that stops an uncharged refusal loop.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ clicks.append(("click", args))
+ raise ToolRefusal(
+ f"ref={args['n']} no longer points to an element",
+ error_class="stale_ref",
+ data={"page_state_changed": True},
+ )
+
+ click = ToolSpec(
+ name="click", description="c", parameters={"type": "object", "properties": {}}, handler=handler, billable=True
+ )
+ script = [[("click", {"n": i})] for i in range(30)]
+ outcome, _ = await _run(
+ script, [click, make_finish_tool()], max_action_steps=1, max_action_steps_ceiling=1, max_turns=40
+ )
+
+ assert len(clicks) == UNCHARGED_REFUSAL_GRACE + 1
+ assert outcome.action_steps == 1
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+
+
@pytest.mark.asyncio
async def test_action_step_budget_extension_not_laundered_by_same_url_reload() -> None:
# A confirmed same-URL navigate reports page_state_changed (the retry ledger legitimately
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index db0ed5c74..60649ecec 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -1900,10 +1900,10 @@ async def test_a_totp_placeholder_not_typed_alone_is_refused_without_blaming_the
# Nothing is wrong with the credential here: the model combined its placeholder with other text or
# made one up. It is refused with words saying so, and the run is not attributed to a missing source.
with _credential_totp_run(monkeypatch, seed=_TOTP_SEED) as run:
- with pytest.raises(taskv3_loop.ToolRefusal) as excinfo:
- await _tool(run.tools, "type").handler({"selector": "#otp", "text": text})
- assert "typed alone" in str(excinfo.value)
- assert "no usable" not in str(excinfo.value) and "BW_TOTP" not in str(excinfo.value)
+ refused = await _tool(run.tools, "type").handler({"selector": "#otp", "text": text})
+ assert refused.refused, refused.content
+ assert "typed alone" in refused.content
+ assert "no usable" not in refused.content and "BW_TOTP" not in refused.content
assert [c for c in run.page.calls if c[0] == "fill"] == []
assert run.state.totp_source_missing is False
assert run.state.values_delivered == 0
@@ -1957,12 +1957,12 @@ async def test_a_url_argument_never_carries_a_totp_placeholder(
with _credential_totp_run(
monkeypatch, seed=_TOTP_SEED, other_credentials={"other_login": _OTHER_TOTP_PLACEHOLDER}
) as run:
- with pytest.raises(taskv3_loop.ToolRefusal) as excinfo:
- await _tool(run.tools, tool_name).handler(dict(args))
- assert "BW_TOTP" not in str(excinfo.value) and _TOTP_PLACEHOLDER not in str(excinfo.value)
+ refused = await _tool(run.tools, tool_name).handler(dict(args))
+ assert refused.refused, refused.content
+ assert "BW_TOTP" not in refused.content and _TOTP_PLACEHOLDER not in refused.content
# The credential's source is fine; the model sent its placeholder somewhere a code never goes.
- assert "never a URL or file" in str(excinfo.value)
- assert "typed alone" not in str(excinfo.value) and "no usable" not in str(excinfo.value)
+ assert "never a URL or file" in refused.content
+ assert "typed alone" not in refused.content and "no usable" not in refused.content
assert [c for c in run.page.calls if c[0] in ("goto", "fill", "set_input_files")] == []
assert fetched == []
assert run.state.values_delivered == 0
@@ -9745,6 +9745,27 @@ async def test_dom_a_tab_strip_is_still_not_a_menu() -> None:
assert "opened a menu of" not in r.content
+ "args",
+ [{"selector": '[data-tv3="t157"]'}, {"mark": 999}],
+ ids=["refused_by_the_tool", "refused_by_the_address_wrapper"],
+)
+async def test_a_download_that_lands_during_a_refused_call_is_still_reported(
+ tmp_path: Path, args: dict[str, Any]
+) -> None:
+ page = _ClickFakePage(exists=False)
+ tools = build_browser_tools(_fixed_page_provider(page), downloads_dir=str(tmp_path))
+ await _dispatch(tools, "click", dict(args))
+ (tmp_path / "statement.pdf").write_bytes(b"x" * 10)
+
+ result, refused = await _dispatch(tools, "click", dict(args))
+
+ assert (result.status, refused) == ("error", True), result.content
+ assert "Downloaded: statement.pdf (10 B)" in result.content
+ assert (result.data or {}).get("download_new") is True
+
+
@pytest.mark.asyncio
async def test_click_stale_marker_fast_fails_without_15s_wait() -> None:
# A [data-tv3=...] marker is minted only by our own enrichment: if it matches nothing now, it can
@@ -9752,8 +9773,8 @@ async def test_click_stale_marker_fast_fails_without_15s_wait() -> None:
# pure loss. Fail fast and loud, and never dispatch the doomed click.
page = _ClickFakePage(exists=False)
tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t157"]'})
- assert r.status == "error"
+ r, refused = await _dispatch(tools, "click", {"selector": '[data-tv3="t157"]'})
+ assert (r.status, refused) == ("error", True)
assert "no longer exists" in r.content
assert "e-observe" in r.content
assert not any(c[0] == "click" for c in page.calls)
@@ -9815,7 +9836,7 @@ async def test_click_on_a_marker_the_page_cloned_never_silently_lands_on_the_clo
assert await page.locator(selector).count() == 2, "fixture is not armed: the clone must carry the marker"
assert await page.locator(selector).first.text_content() == "Remove Beta"
- r = await _tool(tools, "click").handler({"selector": selector})
+ r, _ = await _dispatch(tools, "click", {"selector": selector})
clicked = await page.evaluate("() => window.__clicked")
if r.status == "ok":
@@ -9872,9 +9893,9 @@ async def test_click_on_a_marker_the_page_destroyed_still_fails_loud() -> None:
)
assert await page.locator(selector).count() == 0, "fixture is not armed"
- r = await _tool(tools, "click").handler({"selector": selector})
+ r, refused = await _dispatch(tools, "click", {"selector": selector})
- assert r.status == "error"
+ assert (r.status, refused) == ("error", True)
assert "no longer exists" in r.content and "e-observe" in r.content
assert r.data == {"page_state_changed": True} # a same-document re-render must poison the batch
assert await page.evaluate("() => window.__clicked") == []
@@ -9886,8 +9907,8 @@ async def test_click_recounts_a_marker_that_reattached_as_two_copies_during_the_
# back as two copies satisfies "attached" with both present, and the count must be re-read then.
page = _ClickFakePage(exists=False, match_counts=[0, 2])
tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t7"]'})
- assert r.status == "error"
+ r, refused = await _dispatch(tools, "click", {"selector": '[data-tv3="t7"]'})
+ assert (r.status, refused) == ("error", True)
assert "matches 2 elements" in r.content and "e-observe" in r.content
assert any(c[0] == "wait_for_selector" for c in page.calls)
assert not any(c[0] == "click" for c in page.calls)
@@ -9899,9 +9920,11 @@ async def test_click_timeout_on_vanished_element_reports_removal() -> None:
# surfacing a generic Playwright timeout the model cannot act on.
page = _ClickFakePage(exists=False, click_raises=TimeoutError("Page.click: Timeout 15000ms exceeded"))
tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "click").handler({"selector": "#opt-old"})
+ r, refused = await _dispatch(tools, "click", {"selector": "#opt-old"})
assert r.status == "error"
assert "no longer exists" in r.content
+ # The click was dispatched and may have moved the page first, so it is charged, never a refusal.
+ assert refused is False
@pytest.mark.asyncio
@@ -10357,7 +10380,7 @@ async def test_dom_stale_menu_marker_click_fails_fast_and_loud() -> None:
r2 = await click.handler({"selector": "#sort-trigger"})
assert "CLOSED the open menu" in r2.content
start = time.monotonic()
- r3 = await click.handler({"selector": '[data-tv3-menu="3"]'})
+ r3, _ = await _dispatch([click], "click", {"selector": '[data-tv3-menu="3"]'})
elapsed = time.monotonic() - start
assert r3.status == "error"
assert "no longer exists" in r3.content
@@ -10376,7 +10399,7 @@ async def test_dom_stale_marker_click_fails_fast_and_loud() -> None:
)
tools = build_browser_tools(_fixed_page_provider(page))
start = time.monotonic()
- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t9"]'})
+ r, _ = await _dispatch(tools, "click", {"selector": '[data-tv3="t9"]'})
elapsed = time.monotonic() - start
assert r.status == "error"
assert "no longer exists" in r.content
@@ -10754,6 +10777,300 @@ async def test_observe_names_anonymous_shadow_hosted_controls_through_their_host
assert again_next == by_label["Next"]
+async def _dispatch(tools: list[Any], name: str, args: dict[str, Any]) -> tuple[Any, bool]:
+ """The result the loop records for one call, and whether the tool refused it (an uncharged call)."""
+ result = await _tool(tools, name).handler(args)
+ return result, result.refused
+
+
+# A rail entry and a second copy of it inside a panel. `panel_style` decides whether the copy renders.
+def _two_copy_rail(panel_style: str) -> str:
+ return f"""<nav id="rail"><div class="item" style="width:140px;height:24px">
+ <span>Open Items</span></div></nav>
+ <div id="panel" style="{panel_style}"><div class="item" style="width:140px;height:24px">
+ <span>Open Items</span></div></div>
+ <script>
+ window.__clicked = [];
+ document.querySelector('#rail .item').addEventListener('click', () => window.__clicked.push('rail'));
+ document.querySelector('#panel .item').addEventListener('click', () => window.__clicked.push('panel'));
+ </script>"""
+
+
+@_skip_no_browser
+async def test_a_typed_selector_matching_two_is_refused_before_acting() -> None:
+ async with _live_page(_two_copy_rail("")) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ for tool_name, extra in (("click", {}), ("hover", {}), ("type", {"text": "x"})):
+ result, refused = await _dispatch(tools, tool_name, {"selector": "text=Open Items", **extra})
+ assert (result.error_class, refused) == ("ambiguous_selector", True), (tool_name, result.content)
+ assert "matches 2 elements" in result.content
+ assert await page.evaluate("window.__clicked") == []
+
+
+@_skip_no_browser
+async def test_a_typed_selector_that_matches_nothing_says_so_without_claiming_a_rerender() -> None:
+ async with _live_page(_two_copy_rail("visibility:hidden")) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+
+ result, refused = await _dispatch(tools, "click", {"selector": "text=Closed Items"})
+
+ assert (result.error_class, refused) == ("stale_selector", True), result.content
+ assert "matches nothing on the page" in result.content
+ assert "re-rendered" not in result.content
+ # The flag poisons the rest of the batch and reads as page progress; nothing moved here.
+ assert not (result.data or {}).get("page_state_changed")
+
+
+# Every event a real act on the page would raise, recorded in the capture phase before any handler.
+_TOUCH_RECORDER_JS = """() => {
+ window.__touched = [];
+ for (const type of ['pointerdown', 'mousedown', 'mouseup', 'click', 'mouseover', 'focusin', 'keydown', 'input']) {
+ window.addEventListener(type, (e) => window.__touched.push(type), true);
+ }
+}"""
+
+
+_REFUSAL_PAGE = (
+ _two_copy_rail("")
+ + """<button id="save" style="width:80px;height:20px">Save</button>
+ <div id="gone"><button style="width:80px;height:20px">Remove</button></div>
+ <input id="code" style="width:80px;height:20px">"""
+)
+
+
+async def _observed_marker(tools: list[Any], page: Any, label: str) -> str:
+ await _tool(tools, "observe").handler({})
+ marker = next(e["selector"] for e in (await _observe_data(page))["elements"] if e.get("label") == label)
+ assert marker.startswith('[data-tv3="'), marker
+ return marker
+
+
+async def _clone_remove_button(page: Any) -> None:
+ await page.evaluate(
+ "() => { const b = document.querySelector('#gone button'); b.parentNode.appendChild(b.cloneNode(true)); }"
+ )
+
+
+_RefusalArrange = Callable[[Any, pytest.MonkeyPatch], Awaitable[tuple[list[Any], str, dict[str, Any]]]]
+
+
+async def _typed_two_visible(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ return build_browser_tools(_fixed_page_provider(page)), "click", {"selector": "text=Open Items"}
+
+
+async def _typed_zero_match(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ return build_browser_tools(_fixed_page_provider(page)), "hover", {"selector": "text=Closed Items"}
+
+
+async def _marker_vanished(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ marker = await _observed_marker(tools, page, "Remove")
+ await page.evaluate("() => document.querySelector('#gone').remove()")
+ return tools, "hover", {"selector": marker}
+
+
+async def _marker_cloned(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ marker = await _observed_marker(tools, page, "Remove")
+ await _clone_remove_button(page)
+ return tools, "hover", {"selector": marker}
+
+
+async def _click_marker_vanished(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools, _, args = await _marker_vanished(page, mp)
+ return tools, "click", args
+
+
+async def _click_marker_cloned(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools, _, args = await _marker_cloned(page, mp)
+ return tools, "click", args
+
+
+async def _mark_not_in_latest(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ await _tool(tools, "look").handler({})
+ return tools, "click", {"mark": 999}
+
+
+async def _stale_mark(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ await page.set_content('<button id="only" style="width:80px;height:20px">Only</button>')
+ tools = build_browser_tools(_fixed_page_provider(page))
+ await _tool(tools, "look").handler({})
+ await page.evaluate("() => document.getElementById('only').remove()")
+ return tools, "click", {"mark": 1}
+
+
+async def _invalid_mark(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ return build_browser_tools(_fixed_page_provider(page)), "type", {"mark": "first", "text": "x"}
+
+
+async def _ref_not_in_latest(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ await _tool(tools, "observe").handler({})
+ return tools, "type", {"selector": "ref=999", "text": "x"}
+
+
+async def _stale_ref(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ ref = _ref_line((await _tool(tools, "observe").handler({})).content, "Save")
+ await page.evaluate("() => document.getElementById('save').remove()")
+ return tools, "click", {"selector": ref}
+
+
+async def _selector_in_two_frames(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ mp.setattr(settings, "TASK_V3_FRAME_PERCEPTION", True)
+ frame = '<iframe style="width:200px;height:60px" srcdoc="<input id=pin>"></iframe>'
+ await page.set_content(frame + frame)
+ await page.wait_for_function(
+ "() => [...document.querySelectorAll('iframe')].every(f => f.contentDocument?.getElementById('pin'))"
+ )
+ return build_browser_tools(_fixed_page_provider(page)), "type", {"selector": "#pin", "text": "x"}
+
+
+async def _code_inside_other_text(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page), resolve_typed_text=lambda text: text)
+ return tools, "type", {"selector": "#code", "text": f"code {_TOTP_PLACEHOLDER}"}
+
+
+async def _code_with_no_resolver(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ tools = build_browser_tools(_fixed_page_provider(page), resolve_typed_text=lambda text: "BW_TOTP")
+ return tools, "type", {"selector": "#code", "text": _TOTP_PLACEHOLDER}
+
+
+async def _code_with_no_secret(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ task = SimpleNamespace(task_id="tsk_otp", organization_id="o_1", workflow_run_id=None)
+ tools = build_browser_tools(
+ _fixed_page_provider(page),
+ resolve_typed_text=lambda text: "BW_TOTP",
+ resolve_totp_placeholder=VerificationState(task=task).resolve_totp_placeholder,
+ )
+ return tools, "type", {"selector": "#code", "text": _TOTP_PLACEHOLDER}
+
+
+async def _removed_by_the_click(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:
+ # The pointer reaching the target removes it, so the click is dispatched and then fails as stale.
+ mp.setattr(taskv3_tools, "_ACTION_TIMEOUT_MS", 1000)
+ await page.set_content('<button id="flee" style="width:80px;height:20px" onmouseover="this.remove()">Flee</button>')
+ return build_browser_tools(_fixed_page_provider(page)), "click", {"selector": "#flee"}
+
+
+# An async typeahead: rows appear 150ms after two typed characters, and a row click commits.
+_ASYNC_TYPEAHEAD_PAGE = """<div><input id="loc" role="combobox" aria-autocomplete="list" aria-haspopup="listbox"
+ aria-expanded="false" autocomplete="off" style="width:200px;height:24px"></div>
+ <div id="menu" role="listbox" hidden></div>
+ <div id="pair"><button class="twin" style="width:80px;height:20px">Next</button>
+ <button class="twin" style="width:80px;height:20px">Next</button></div>
+ <script>
+ const loc = document.getElementById('loc'), menu = document.getElementById('menu');
+ loc.addEventListener('input', () => {
+ const q = loc.value.trim();
+ setTimeout(() => {
+ if (loc.value.trim() !== q || q.length < 2) return;
+ menu.innerHTML = '';
+ ['San Diego, California', 'San Jose, California'].filter((o) => o.includes(q)).forEach((label) => {
+ const row = document.createElement('div');
+ row.setAttribute('role', 'option');
+ row.style.height = '30px';
+ row.textContent = label;
+ row.addEventListener('mousedown', (e) => e.preventDefault());
+ row.addEventListener('click', () => { loc.value = label; menu.hidden = true; });
+ menu.appendChild(row);
+ });
+ menu.hidden = false;
+ loc.setAttribute('aria-expanded', 'true');
+ }, 150);
+ });
+ </script>"""
+
+
+async def _ambiguous_after_releasing_a_live_row(
+ page: Any, mp: pytest.MonkeyPatch
+) -> tuple[list[Any], str, dict[str, Any]]:
+ # The partial value leaves its typeahead row offered; the next call first clears that field.
+ await page.set_content(_ASYNC_TYPEAHEAD_PAGE)
+ tools = build_browser_tools(_fixed_page_provider(page))
+ offered = await _tool(tools, "select_combobox").handler({"selector": "#loc", "value": "San Diego"})
+ assert "data-tv3-pick" in offered.content, offered.content
+ return tools, "click", {"selector": "#pair .twin"}
+
+
+@_skip_no_browser
+ "arrange,refused",
+ [
+ (_typed_two_visible, True),
+ (_typed_zero_match, True),
+ (_marker_vanished, True),
+ (_marker_cloned, True),
+ (_click_marker_vanished, True),
+ (_click_marker_cloned, True),
+ (_mark_not_in_latest, True),
+ (_stale_mark, True),
+ (_invalid_mark, True),
+ (_ref_not_in_latest, True),
+ (_stale_ref, True),
+ (_selector_in_two_frames, True),
+ (_code_inside_other_text, True),
+ (_code_with_no_resolver, True),
+ (_code_with_no_secret, True),
+ (_removed_by_the_click, False),
+ (_ambiguous_after_releasing_a_live_row, False),
+ ],
+ ids=lambda v: v.__name__.strip("_") if callable(v) else ("refused" if v else "charged"),
+)
+async def test_every_uncharged_refusal_is_issued_before_the_page_is_touched(
+ arrange: _RefusalArrange, refused: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # The loop does not charge a refused call because it cannot have moved the page. Each case drives
+ # one ToolRefusal raiser through the real loop; the last one is a click that dispatched and then
+ # failed, which must stay charged, since it did touch the page.
+ from skyvern.forge.agent import _PAGE_FINGERPRINT_PROBE_JS # noqa: PLC0415
+
+ async with _live_page(_REFUSAL_PAGE) as page:
+ tools, tool_name, args = await arrange(page, monkeypatch)
+ # Parked in an empty corner and left to settle before arming: Chromium fires a synthetic mouseover
+ # when layout changes under a resting pointer, which is the page's doing, not the tool's.
+ await page.mouse.move(1010, 890)
+ # A page that renders no frames (a background or throttled tab) never runs rAF, so the wait is bounded.
+ await page.evaluate(
+ "() => new Promise((r) => { requestAnimationFrame(() => requestAnimationFrame(r)); setTimeout(r, 200); })"
+ )
+ await page.evaluate(_TOUCH_RECORDER_JS)
+ before = (page.url, await page.evaluate(_PAGE_FINGERPRINT_PROBE_JS))
+ rounds: list[list[taskv3_loop.RoundAction]] = []
+
+ async def _on_round(actions: list[taskv3_loop.RoundAction], _text: str | None) -> None:
+ rounds.append(list(actions))
+
+ outcome = await taskv3_loop.run_agent_tool_loop(
+ llm_caller=_ScriptedCaller([[(tool_name, args)], [("finish", {"status": "failed", "reason": "x"})]]),
+ system_prompt="sys",
+ user_prompt="goal",
+ tools=tools + [taskv3_loop.make_finish_tool()],
+ max_turns=5,
+ max_tool_calls=5,
+ max_action_steps=1,
+ max_action_steps_ceiling=1,
+ on_action_round=_on_round,
+ )
+
+ [[action]] = rounds
+ assert not action.succeeded, action
+ if refused:
+ assert not action.billable, action.error
+ elif arrange is _removed_by_the_click:
+ assert "no longer exists" in (action.error or ""), "the click did not dispatch and then go stale"
+ else:
+ assert "matches 2 elements" in (action.error or ""), action.error
+ if not action.billable:
+ assert outcome.action_steps == 0
+ assert await page.evaluate("window.__touched") == [], action.error
+ assert (page.url, await page.evaluate(_PAGE_FINGERPRINT_PROBE_JS)) == before, action.error
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_click_refuses_a_host_anchored_selector_the_page_has_since_cloned() -> None:
@@ -10798,7 +11115,7 @@ async def test_click_refuses_a_host_anchored_selector_the_page_has_since_cloned(
("select_combobox", {"value": "x"}),
("hover", {}),
):
- cr = await _tool(tools, tool_name).handler({"selector": sel, **extra})
+ cr, _ = await _dispatch(tools, tool_name, {"selector": sel, **extra})
assert cr.status == "error", (tool_name, cr.content)
assert "matches 2 elements" in cr.content, (tool_name, cr.content)
assert await page.evaluate("window.__clicked") == []
@@ -16329,7 +16646,7 @@ async def test_no_poisoned_dispatch_can_pair_a_digest_line_with_a_decoy_element(
# Non-vacuous: the attack must not simply hide Alpha, or there would be nothing to mis-address.
alpha_ref = _ref_line(r.content, "'Alpha'")
- clicked = await _tool(tools, "click").handler({"selector": alpha_ref})
+ clicked, _ = await _dispatch(tools, "click", {"selector": alpha_ref})
hits = await page.evaluate("() => window.hits")
assert "beta" not in hits, (attack, clicked.content, r.content)
if clicked.status == "ok":
@@ -16380,7 +16697,7 @@ async def test_a_ref_does_not_survive_a_navigation_onto_a_same_tag_lookalike() -
await page.goto("data:text/html,<button id='go' style='width:90px;height:22px'>Elsewhere</button>")
assert await page.locator("#go").count() == 1, "fixture must offer exactly one look-alike"
- acted = await _tool(tools, "click").handler({"selector": ref})
+ acted, _ = await _dispatch(tools, "click", {"selector": ref})
assert acted.status == "error", acted.content
@@ -16413,7 +16730,7 @@ async def test_a_live_handle_is_never_displaced_by_a_selector_that_drifted_onto_
assert await page.locator("#go").count() == 1, "fixture must leave the selector naming ONE element"
assert await page.locator("#go").first.text_content() == "Other", "and that element must be the other one"
- acted = await _tool(tools, "click").handler({"selector": ref})
+ acted, _ = await _dispatch(tools, "click", {"selector": ref})
assert "Other" not in await page.evaluate("() => window.hits"), acted.content
assert acted.status == "error", acted.content
@@ -16444,7 +16761,7 @@ async def test_a_ref_whose_element_was_rebuilt_into_twins_is_refused_rather_than
await page.evaluate("window.__dup()")
assert await page.locator("#go").count() == 2, "fixture must arm the ambiguity"
- cr = await _tool(tools, "click").handler({"selector": ref})
+ cr, _ = await _dispatch(tools, "click", {"selector": ref})
assert cr.status == "error", cr.content
assert "re-render" in cr.content or "re-observe" in cr.content, cr.content
assert await page.evaluate("() => window.hits") == []
@@ -22151,7 +22468,7 @@ async def test_a_hijacked_setattribute_cannot_hand_back_a_selector_naming_anothe
" if (n === 'data-tv3-act') { return set.call(document.getElementById('decoy'), n, v); }"
" return set.call(this, n, v); }; }"
)
- result = await _tool(tools, "click").handler({"mark": 1})
+ result, _ = await _dispatch(tools, "click", {"mark": 1})
assert result.status == "error", result.content
assert "no longer points to an element" in result.content
@@ -22182,7 +22499,7 @@ async def test_a_setattribute_trap_cannot_hand_back_a_token_two_elements_carry()
" const out = Array.from(qsa.call(this, sel));"
" return out.filter((e) => e.id !== 'decoy'); }; }"
)
- result = await _tool(tools, "click").handler({"mark": 1})
+ result, _ = await _dispatch(tools, "click", {"mark": 1})
assert result.status == "error", result.content
assert "no longer points to an element" in result.content
@@ -23426,7 +23743,7 @@ async def test_click_on_stale_suggestion_marker_fails_fast_with_reobserve_error(
# as owned markers: an absent one gets the fast marker error, not a full actionability timeout.
async with _content_page(_duplicate_suggestion_html(_DUPLICATE_STREET_ROWS)) as page:
tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "click").handler({"selector": '[data-tv3-sugg="7"]'})
+ r, _ = await _dispatch(tools, "click", {"selector": '[data-tv3-sugg="7"]'})
assert r.status == "error", r.content
assert "markers vanish" in r.content, r.content
@@ -31173,9 +31490,9 @@ async def test_a_ref_refuses_when_the_reading_could_not_say_which_document_it_ca
assert reading.status == "ok", reading.content
ref = _ref_line(reading.content, "First name")
- result = await _tool(tools, "click").handler({"selector": ref})
+ result, refused = await _dispatch(tools, "click", {"selector": ref})
- assert result.status == "error", result.content
+ assert (result.status, refused) == ("error", True), result.content
assert "re-observe" in result.content.lower() or "observe()" in result.content, result.content
# And it really did not act: no click reached the page for that ref.
assert not any(call[0] == "click" for call in page.calls), page.calls
@@ -31402,7 +31719,7 @@ async def test_every_exit_from_address_resolution_records_exactly_one_stamped_re
# An address that fails to resolve still spent real time looking.
readings.clear()
- failed = await tools["click"].handler({"selector": "ref=1"})
+ failed, _ = await _dispatch(list(tools.values()), "click", {"selector": "ref=1"})
assert failed.status == "error"
assert len(readings) == 1
@@ -31424,7 +31741,7 @@ async def test_every_exit_from_address_resolution_records_exactly_one_stamped_re
# which definition produced it, which is the one thing the contract says cannot happen.
readings.clear()
stamps.clear()
- rejected = await tools["click"].handler({"mark": "not-an-integer"})
+ rejected, _ = await _dispatch(list(tools.values()), "click", {"mark": "not-an-integer"})
assert rejected.status == "error"
assert len(readings) == 1
assert stamps == [False]