Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop charging Task V3 action steps for calls refused before acting on the page (#8688)
pedrohsdb6a88ba618305
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex d99830a6d..e5e8864df 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -201,6 +201,12 @@ class ToolResult:# OTHER one drops it silently; the classmethods below cannot express that pairing (neither# accepts the other's kwarg) and the raw constructor is the only route that could.ok_class: ToolOkClass | None = None+ # The requested action was refused and never dispatched (a ToolRefusal). A field, not a raise, so+ # every wrapper around the handler still post-processes the call.+ refused: bool = False+ # A refused call that still dispatched input to the page first, to release a previous field's+ # suggestion list. It is charged and the page may have changed, though the requested action never ran.+ touched_page: bool = False@classmethoddef ok(@@ -224,12 +230,21 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]class ToolRefusal(Exception):- """Raised below a handler's return path to refuse the call; the loop sends the message alone as- the tool's error, so it must be written for the model and carry nothing it must not see."""+ """Refuse the call before it acts on the page (no input dispatched; a marker attribute written to+ address an element is not an action). The loop sends the message alone as a refused error, which+ does not spend an action step within the grace, so it must carry nothing the model must not see."""- def __init__(self, message: str, *, error_class: ToolErrorClass) -> None:+ def __init__(self, message: str, *, error_class: ToolErrorClass, data: dict[str, Any] | None = None) -> None:super().__init__(message)self.error_class: ToolErrorClass = error_class+ self.data = data++ @classmethod+ def of(cls, result: ToolResult) -> ToolRefusal:+ return cls(result.content, error_class=result.error_class or "other", data=result.data)++ def as_result(self) -> ToolResult:+ return ToolResult("error", str(self), self.data, error_class=self.error_class, refused=True)# The tool-result `data` keys the target-name/target-kind capture ride on (written by the browser@@ -677,6 +692,9 @@ ACTION_BUDGET_EXTENSION_EVIDENCE_WINDOW = 8# Total growth stops at this multiple of the original cap. The wall clock — the one cap that is not# a function of the action-step budget — is the outer runaway stop and is never re-derived.ACTION_BUDGET_EXTENSION_MAX_FACTOR = 3+# Refusals left uncharged in a row before a refusal is charged like any failed call. The repeat guard cannot+# bound them (a stale address resets it), so this does.+UNCHARGED_REFUSAL_GRACE = ACTION_LOOP_NUDGE_AFTER# Facetable event names — both the grant and the refusal are queryable so the gate's decision# precision is measurable on the canary; change only with the dashboards that read them.ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"@@ -2962,6 +2980,8 @@ class LoopState:tool_seconds: float = 0.0total_tokens: int = 0action_steps: int = 0+ # Refused billable calls left uncharged since the last charged, non-refused one.+ uncharged_refusals: int = 0budget_extended_notice: str | None = None# Progress-gated budget extension (SKY-15264, SKY-15666): how many extensions have been granted,# and the cap they are all sized and bounded against — captured before the first grant so growth@@ -3604,7 +3624,10 @@ async def run_agent_tool_loop(st.messages.append({"role": "user", "content": NO_TOOL_CALL_NUDGE})continue- turn_did_action = False+ # Dispatched feeds the page-state stall detector; charged feeds the action-step budget and the+ # rows' billable flag. They differ only by calls the tool refused before acting on the page.+ turn_dispatched_billable = False+ turn_charged = Falsest.stall_nudges_due = []st.refresh_nudge_due = Falsest.budget_extended_notice = None@@ -3629,7 +3652,7 @@ async def run_agent_tool_loop(batch_page_change_reason: str | None = Nonebatch_fp_before: str | None = None# Sample only when the batch can actually land a billable action -- the end-of-batch check- # below gates on turn_did_action, so a finish-only or perception-only batch has no use for+ # below gates on turn_dispatched_billable, so a finish-only or perception-only batch has no use for# this baseline and shouldn't pay its round-trip.batch_has_billable_call = any(tool_by_name.get(tool_name) is not None and tool_by_name[tool_name].billable@@ -3651,7 +3674,7 @@ async def run_agent_tool_loop(# The page moved BETWEEN batches (a delayed render landing after the prior# after-sample): the touches the old samples described are stale, and this batch's# dispatch and extension decisions must not read them. Canonical-only — the- # incumbent stall counters keep their end-of-batch turn_did_action gate.+ # incumbent stall counters keep their end-of-batch turn_dispatched_billable gate.st.canonical.progress(_ProgressEvidence.CROSS_BATCH_MOVEMENT)batch_fp_after: str | None = Noneif activity is not None:@@ -4084,13 +4107,11 @@ async def run_agent_tool_loop(result = ToolResult.error(f"unknown_tool: {tool_name}")else:if spec.billable:- # A dispatched page action consumes a step even if it errors (it may mutate before- # failing); billing below counts successes only.- turn_did_action = True+ turn_dispatched_billable = Truetry:result = await spec.handler(args)except ToolRefusal as refusal:- result = ToolResult.error(str(refusal), error_class=refusal.error_class)+ result = refusal.as_result()except Exception as exc:LOG.warning("taskv3 tool handler raised",@@ -4102,6 +4123,16 @@ async def run_agent_tool_loop(result = ToolResult.error(f"tool_error: {type(exc).__name__}: {exc}", error_class=raised_class)tool_duration_seconds = time.monotonic() - tool_started_atst.tool_seconds += tool_duration_seconds+ # A dispatched page action consumes a step even if it errors (it may mutate before failing),+ # unless the tool refused it before acting on the page, within the grace. Billing counts+ # successes only.+ call_charged = spec is not None and spec.billable+ if call_charged and result.status == "error" and result.refused and not result.touched_page:+ call_charged = st.uncharged_refusals >= UNCHARGED_REFUSAL_GRACE+ st.uncharged_refusals += 0 if call_charged else 1+ elif call_charged:+ st.uncharged_refusals = 0+ turn_charged = turn_charged or call_charged# Entry is recorded only when the tool succeeded (a stale-ref failure put nothing in the# field), but the submit is recorded on DISPATCH whatever the verdict: the loop ran the# click itself, so nothing here asks the page whether a submission completed. Entry first,@@ -4133,6 +4164,10 @@ async def run_agent_tool_loop(# vocabulary with Python exception names under one facet, and taskv3 emits on every# erroring tool call so it would dominate the values.cost_fields["tool_error_class"] = result.error_class or "other"+ # Only on the rows it describes, like the fields below: a billable call that did not+ # spend an action step because the tool refused it before acting on the page.+ if spec is not None and spec.billable and not call_charged:+ cost_fields["charged"] = False# Only on the class they describe, so every other erroring row keeps exactly the# fields it has today. Total over `covered` rows by construction: the helper that# builds all three messages records before it returns any of them, so a covered row@@ -4317,7 +4352,7 @@ async def run_agent_tool_loop(status=result.status,content=model_facing_content,perception=spec.compactable,- page_changing=spec.touches_page,+ page_changing=spec.touches_page and (result.touched_page or not result.refused),# The two ways a secret reaches the page: a credential placeholder typed by an# entry tool, and a value a tool registered as secret (a delivered verification# code) for the model to type next.@@ -4397,8 +4432,9 @@ async def run_agent_tool_loop(if spec is not None and spec.billable:if st.progress is not None:st.progress.on_billable()- # Errored dispatches count too: a failed attempt consumed a step (see the action-step- # accounting above) and a repeat-failing action is the same no-progress pathology.+ # Errored and refused dispatches count too: a repeat-failing action is the same+ # no-progress pathology. It does not bound refusals (a stale one resets it);+ # UNCHARGED_REFUSAL_GRACE does.repeat_count, first_turn, streak_fp, streak_moved = st.action_counts.get(action_key, (0, st.turns, batch_fp_before, False))@@ -4459,9 +4495,9 @@ async def run_agent_tool_loop(elif spec.compactable and result.status == "ok":activity.perceptions += 1if spec is not None and (spec.billable or spec.recordable):- # Dispatched page actions enter the round with their outcome: a failed billable round+ # Dispatched page actions enter the round with their outcome: a failed charged round# still consumed budget and must persist (else later blocks undercount the run- # budget); recordable tools persist for artifact parity without billing/budget.+ # budget); refused calls and recordable tools persist without claiming a budget unit.round_outcome = result_data.get(ACTION_OUTCOME_DATA_KEY)round_outcome = round_outcome if isinstance(round_outcome, dict) else Noneround_actions.append(@@ -4471,7 +4507,7 @@ async def run_agent_tool_loop(result.status == "ok" and not _outcome_reports_failure(round_outcome),result_data.get(TARGET_LABEL_DATA_KEY) or None,result_data.get(TARGET_KIND_DATA_KEY) or None,- spec.billable,+ call_charged,round_outcome,result.content if result.status == "error" else None,)@@ -4649,7 +4685,12 @@ async def run_agent_tool_loop(# leaves the rendered document byte-identical ticks the counter, whatever tools produced it.# A missing sample is no evidence either way; any page-change flag or fingerprint movement# re-baselines.- if st.outcome is None and turn_did_action and page_fingerprint is not None and batch_fp_before is not None:+ if (+ st.outcome is None+ and turn_dispatched_billable+ and page_fingerprint is not None+ and batch_fp_before is not None+ ):if st.page_state_prev_fp is not None and batch_fp_before != st.page_state_prev_fp:# The page moved BETWEEN batches (a delayed render landing after the prior# after-sample): the streak the old samples described is stale.@@ -4783,7 +4824,7 @@ async def run_agent_tool_loop(# turns (observe/get_html) don't consume the caller's step budget — the step engine bundles# perception into each step, so counting v3's perception rounds against the same budget# under-counts equivalent work.- if turn_did_action:+ if turn_charged:st.action_steps += 1# Hand the round's executed actions to the caller so it can persist per-action artifacts# (screenshot, DB rows) — kept out of this transport-agnostic core, like should_cancel. Adiff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 8b89d234e..b52465092 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -15,6 +15,7 @@ from __future__ import annotationsimport asyncioimport contextlibimport dataclasses+import functoolsimport ioimport jsonimport os@@ -357,6 +358,8 @@ def _escape_tags_in_text(text: str) -> str:_TV3_MARKER_SELECTOR_RE = re.compile(r'^\[data-tv3(?:(?:-menu|-act)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$')+# Any v3 marker inside a selector, including one observe composed a host-anchored selector around.+_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg)?="')# An opaque identifier (a uuid, or a run of 12+ hex digits) does not survive a model's copy: one# transposed pair sends every later call to a selector that matches nothing. observe addresses its# own elements by ref for that reason; this is what look()'s legend refuses to use as a label.@@ -471,6 +474,20 @@ def _inert_target_error(selector: str) -> ToolResult:)+def _with_refusal_result(handler: ToolHandler) -> ToolHandler:+ """Turn a raised ToolRefusal into its refused result, so the wrappers outside this one post-process+ a refused call (download notices, frame work, typeahead release) like any other."""++ @functools.wraps(handler)+ async def wrapped(args: dict[str, Any]) -> ToolResult:+ try:+ return await handler(args)+ except ToolRefusal as refusal:+ return refusal.as_result()++ return wrapped++def _with_selector_guard(handler: ToolHandler, diagnose_inert: InertTargetDiagnosis | None = None) -> ToolHandler:"""Shared seam for selector tools: normalize a bare invalid `#id` before the handler resolves it, andconvert a residual invalid-selector crash into an actionable error instead of a batch-aborting raise."""@@ -11211,29 +11228,36 @@ def build_browser_tools(except Exception:return 1 # count unavailable → do not block, mirroring _marker_matches' fail-open- async def _ambiguous_selector_error(page: Any, selector: str) -> ToolResult | None:+ async def _require_single_target(page: Any, selector: str) -> None:+ """Raise ToolRefusal, before acting on the page, when a counted selector names no one element."""# A host-anchored selector straddles a shadow boundary, which the per-root marker count# cannot see through; the executor's own engine can, so it supplies the count. Playwright's# actions are non-strict and would otherwise land on whichever match comes first.if not (_is_host_anchored_selector(selector) or _TV3_MARKER_SELECTOR_RE.match(selector.strip())):- return None+ returntry:matches = await page.locator(selector).count()except Exception:# Left open, as the marker count is: refusing here would block every action on a page# whose engine hiccups, and the action's own actionability wait still applies.LOG.warning("taskv3 selector count unavailable; acting unverified", selector=selector)- return None+ returnif matches == 1:- return None- if matches == 0:- return ToolResult.error(+ return+ names_marker = bool(_TV3_MARKER_ANYWHERE_RE.search(selector))+ if matches == 0 and names_marker:+ raise ToolRefusal(f"{selector} no longer matches anything on the page — the page re-rendered since it was ""observed. Re-observe and act on fresh selectors from the new observation.",data={"page_state_changed": True},error_class="stale_selector",)- return ToolResult.error(+ if matches == 0:+ raise ToolRefusal(+ f"{selector} matches nothing on the page. Re-observe and act on a ref from the observation.",+ error_class="stale_selector",+ )+ raise ToolRefusal(f"{selector} matches {matches} elements, so it does not identify one control. Re-observe and ""act on a selector from the new observation, or narrow this one until it matches exactly one.",error_class="ambiguous_selector",@@ -11488,7 +11512,7 @@ def build_browser_tools(try:await page.wait_for_selector(selector, state="attached", timeout=1200)except Exception:- return ToolResult.error(+ raise ToolRefusal(f"{selector} no longer exists on the page — element markers vanish when the ""page re-renders (a closed menu destroys its options). Re-observe and act on ""fresh selectors from the new observation.",@@ -11500,7 +11524,7 @@ def build_browser_tools(if matches > 1:# A clone of the marked element carries the same marker; the click would silently# land on whichever comes first in document order, so refuse before dispatching it.- return ToolResult.error(+ raise ToolRefusal(f"{selector} now matches {matches} elements — the page re-rendered and cloned the ""marked element, so the marker no longer identifies one control. Re-observe and act ""on fresh selectors from the new observation.",@@ -11508,9 +11532,7 @@ def build_browser_tools(error_class="ambiguous_selector",)else:- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)pre: dict[str, Any] | None = Nonetry:pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))@@ -11930,9 +11952,7 @@ def build_browser_tools(if error is not None:return errorselector = args["selector"]- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)await page.hover(selector, timeout=_ACTION_TIMEOUT_MS)return ToolResult.ok(f"hovered {selector}")@@ -12890,6 +12910,7 @@ def build_browser_tools(await _resolve_mirrored_host_control(page, selector) if isinstance(selector, str) and selector else None)on_field: list[tuple[str, _LiveRowOffer]] = []+ released_input = Falsefor field, offer in list(_live_row_offers.items()):try:relation = await offer.page.evaluate(@@ -12961,11 +12982,14 @@ def build_browser_tools():continue# A page can replace the field during cleanup; writes and Escape must stay on the original node.+ # released_input is set only once a dispatch returned, so a fill that raised first stays uncharged.if current:await offer.element.fill("", timeout=_ACTION_TIMEOUT_MS)+ released_input = Trueif await offer.element.evaluate(read_value) != "":continuedismissed = await _close_lingering_typeahead_list(offer.page, field, None, element=offer.element)+ released_input = Trueif dismissed is not None:return ToolResult.error("The requested action did not run. Closing the previous field's still-open list with "@@ -12981,6 +13005,9 @@ def build_browser_tools(result = await handler(args)finally:_prefetched_page.clear()+ if released_input and result.refused:+ # Releasing the previous field already acted on the page, so this call is charged.+ result = dataclasses.replace(result, touched_page=True)if result.status == "ok" and tool_name in {"select_combobox", "type"}:for field, offer in on_field:if _live_row_offers.get(field) is offer:@@ -13766,9 +13793,7 @@ def build_browser_tools(selector = args.get("selector")if not selector:return ToolResult.error("type needs a selector, or mark=N from the last look().")- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)selector = await _resolve_mirrored_host_control(page, selector)text = await _resolve_text(args.get("text", ""), operation="type", page=page, selector=selector)press_enter = args.get("press_enter")@@ -15212,9 +15237,7 @@ def build_browser_tools(value = args.get("value")label_list = _option_str_list(args.get("labels"))value_list = _option_str_list(args.get("values"))- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)selector = await _resolve_mirrored_host_control(page, selector)try:probe = await _probe_evaluate(page, _SELECT_VISIBILITY_JS, selector, await _probe_arg(page, selector))@@ -15342,9 +15365,7 @@ def build_browser_tools(# The tab's URL, like click's: the re-ask's URL rule reads where a key press that may submit started.url_before = await _url(_current_page())if selector:- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)await page.press(selector, key)else:# Page-level, not realm-level: a Frame has no `keyboard`, and an unaddressed keypress goes@@ -15360,9 +15381,7 @@ def build_browser_tools(return errorselector = args.get("selector")if selector:- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)el = await page.query_selector(selector)if el:await el.scroll_into_view_if_needed()@@ -15555,9 +15574,7 @@ def build_browser_tools(if error is not None:return errorselector = args["selector"]- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)# Validate the selector before fetching: an invalid or missing selector fails here, before anything# is staged into downloads_dir, so the selector guard's residual error can never leave a phantom# upload for the download-signal wrapper to misread as a browser download.@@ -15735,9 +15752,7 @@ def build_browser_tools(if error is not None:return errorselector = args["selector"]- ambiguous = await _ambiguous_selector_error(page, selector)- if ambiguous is not None:- return ambiguous+ await _require_single_target(page, selector)selector = await _resolve_mirrored_host_control(page, selector)offer = _live_row_offers.get(selector)live_offer = None@@ -16065,7 +16080,7 @@ def build_browser_tools(# Its neighbour above ("either mark or selector, not both") deliberately keeps no# class -- that is a schema error with no counterpart in the css cohort, so naming# it would add a value to one side of the comparison this record exists to support.- return ToolResult.error(+ raise ToolRefusal(f"mark must be an integer from the last look(), got {mark!r}.",error_class="invalid_mark",)@@ -16081,7 +16096,7 @@ def build_browser_tools(# rather than hiding exactly the failure cost this field exists to price.selector, mark_error = await _resolve_mark(page, mark_int)if mark_error is not None:- return mark_error+ raise ToolRefusal.of(mark_error)# In place rather than into a copy: everything downstream reads this dict AFTER dispatch# -- the persisted action's element_id, the submit watch, the repeat guard's key and the# nudge's target -- and a copy leaves every one of them seeing only `mark`.@@ -16438,7 +16453,7 @@ def build_browser_tools(return errorrealm, realm_error = await _realm_for_typed_selector(page, selector)if realm_error is not None:- return realm_error+ raise ToolRefusal.of(realm_error)# The routed page is handed on, because the realm decision was made ABOUT it: the# provider is must_get_working_page and can switch to a newer tab, so letting preflight# resolve again would run the selector on a popup that became valid during the frame@@ -16468,7 +16483,7 @@ def build_browser_tools(ref = int(match.group(1))resolved, ref_error = await _resolve_ref(page, ref)if ref_error is not None:- return ref_error+ raise ToolRefusal.of(ref_error)_owner = (_observe_manifest.get(ref) or {}).get("owner")# In place, for the same reason act-by-mark does it: the persisted action's element_id,# the submit watch, the repeat guard's key and the nudge all read this dict AFTER dispatch.@@ -16721,6 +16736,8 @@ def build_browser_tools(# "not in the current set of marks" (a clean no-op), so click/type need no further change.tools = [t for t in tools if t.name != "look"]for _tool_spec in tools:+ # Innermost, so every wrapper below sees a refused call as a result.+ _tool_spec.handler = _with_refusal_result(_tool_spec.handler)if _tool_spec.name in ("click","hover",@@ -16768,6 +16785,8 @@ def build_browser_tools(# args["selector"], so the whole verified click/type path (uniqueness gate, commit-verify)# runs on the act-by-mark selector unchanged._tool_spec.handler = _with_act_by_mark(_tool_spec.handler)+ # Again outermost, for the refusals the mark and ref resolvers raise themselves.+ _tool_spec.handler = _with_refusal_result(_tool_spec.handler)if _tool_spec.name == "click":_tool_spec.toggle_probe = _click_targets_toggle_apply_download_signal(tools, downloads_dir)diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex b0e176470..03bf77fef 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -4877,6 +4877,34 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(]+async def test_execute_task_v3_refused_round_persists_a_failed_row_without_a_budget_unit(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The loop marks a call refused before it touched the page with billable=False and does not count+ # it in action_steps. The pool counts distinct (task_id, step_order) pairs, so a refused round has+ # to ride an index a charged round claims: 2 charged rounds here, so exactly 2 pairs.+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click", "click"])+ refused = RoundAction("click", {"selector": "text=Open Items"}, False, billable=False, error="matches 2 elements")+ rounds = [+ [refused],+ [RoundAction("click", {"selector": "#a"}, True, billable=True)],+ [refused],+ [RoundAction("click", {"selector": "#b"}, True, billable=True)],+ ]+ await _run_execute_task_v3(+ monkeypatch, outcome, action_rounds=rounds, data_extraction_goal=None, extracted_information_schema=None+ )+ action_rows = agent_module.app.DATABASE.workflow_params.create_action.await_args_list[:-1]+ stamped = [(c.kwargs["action"].status, c.kwargs["action"].step_order) for c in action_rows]+ assert stamped == [+ (ActionStatus.failed, 0),+ (ActionStatus.completed, 0),+ (ActionStatus.failed, 0),+ (ActionStatus.completed, 1),+ ]++@pytest.mark.asyncioasync def test_execute_task_v3_failed_run_carries_failure_category(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_multi_field_totp.py b/tests/unit/test_multi_field_totp.pyindex e218d2063..c2d0fd094 100644--- a/tests/unit/test_multi_field_totp.py+++ b/tests/unit/test_multi_field_totp.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport asyncioimport hashlib+import inspectimport jsonimport shutilimport subprocess@@ -867,8 +868,9 @@ async def test_multi_field_totp_browser_privacy_after_fill(monkeypatch: pytest.Massert len(legend) == 6assert [line.split("input ", 1)[1] for line in legend] == [repr("*" * len(value)) for value in values]# The same sanitized labels are retained for subsequent mark actions.+ raw_look = inspect.unwrap(look.handler)look_state = dict(- zip(look.handler.__code__.co_freevars, [cell.cell_contents for cell in look.handler.__closure__])+ zip(raw_look.__code__.co_freevars, [cell.cell_contents for cell in raw_look.__closure__]))assert [look_state["_look_manifest"][i]["label"] for i in range(1, 7)] == ["*" * len(value) for value in valuesdiff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 325490817..4c789f499 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport asyncioimport copy+import dataclassesimport hashlibimport jsonimport random@@ -62,6 +63,7 @@ from skyvern.forge.taskv3.loop import (PERCEPTION_STALL_TERMINATE_AFTER,PROGRESS_LEDGER_SHADOW_EVENT,PROGRESS_LEDGER_WINDOW,+ UNCHARGED_REFUSAL_GRACE,VERDICT_URL_MAX_CHARS,ActivityRecency,LoopOutcome,@@ -69,6 +71,7 @@ from skyvern.forge.taskv3.loop import (SemanticCommitStats,SubmitWatch,ToolHandler,+ ToolRefusal,ToolResult,ToolSpec,_arms_failure_evidence,@@ -2478,6 +2481,128 @@ async def test_action_step_budget_extension_truncated_to_workflow_run_ceiling()assert extended and extended[0]["extension"] == 1+def _click_then(first: ToolResult | ToolRefusal, sink: list[tuple[str, dict[str, Any]]]) -> ToolSpec:+ # The first call answers `first` (returned, or raised when it is a refusal); every later one lands.+ async def handler(args: dict[str, Any]) -> ToolResult:+ sink.append(("click", args))+ if len(sink) == 1:+ if isinstance(first, ToolRefusal):+ raise first+ return first+ return ToolResult.ok("clicked")++ return ToolSpec(+ name="click",+ description="click",+ parameters={"type": "object", "properties": {}},+ handler=handler,+ billable=True,+ )+++async def _atomic_block_run(+ first: ToolResult | ToolRefusal,+) -> tuple[LoopOutcome, int, list[Any], list[list[RoundAction]]]:+ # An atomic Action block: one action step and no extension (the ceiling pins it to the cap).+ clicks: list[tuple[str, dict[str, Any]]] = []+ rounds: list[list[RoundAction]] = []++ async def _on_round(round_actions: list[RoundAction], _text: str | None) -> None:+ rounds.append(list(round_actions))++ script = [+ [("click", {"selector": "text=Open Items"})],+ [("click", {"selector": "ref=3"})],+ [("finish", {"status": "completed", "reason": "done"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [_click_then(first, clicks), make_finish_tool()],+ max_action_steps=1,+ max_action_steps_ceiling=1,+ on_action_round=_on_round,+ )+ assert clicks[0] == ("click", {"selector": "text=Open Items"})+ records = [e for e in logs if e["event"] == "taskv3 tool call finished" and e["tool"] == "click"]+ return outcome, len(clicks), records, rounds+++async def test_a_refusal_before_the_page_is_touched_does_not_spend_the_atomic_blocks_one_round() -> None:+ refusal = ToolRefusal("text=Open Items matches 2 elements", error_class="ambiguous_selector")+ outcome, dispatched, records, rounds = await _atomic_block_run(refusal)++ assert dispatched == 2, "the real click after the refusal was skipped as over budget"+ assert outcome.status == "completed"+ assert outcome.cap_trip is None+ assert outcome.action_steps == 1+ # The workflow-run pool counts rounds off RoundAction.billable: it must agree with action_steps.+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, False)], [(True, True)]]+ assert records[0]["tool_error_class"] == "ambiguous_selector"+ assert records[0]["charged"] is False+ assert "charged" not in records[1]+++async def test_an_error_after_the_act_still_spends_the_atomic_blocks_one_round() -> None:+ # Shaped like a click whose page.click landed and re-rendered the target away: the tool may have+ # mutated the page, so nothing but a declared refusal goes uncharged.+ post_act = ToolResult.error(+ "click on text=Open Items failed: the element no longer exists on the page",+ data={"page_state_changed": True},+ error_class="stale_selector",+ )+ outcome, dispatched, records, rounds = await _atomic_block_run(post_act)++ assert dispatched == 1+ assert outcome.cap_trip == "Reached the maximum steps (1)"+ assert outcome.action_steps == 1+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, True)]]+ assert "charged" not in records[0]+++async def test_a_refusal_that_touched_the_page_first_spends_the_atomic_blocks_one_round() -> None:+ # The tool released another field's suggestion list before refusing: input reached the page.+ touched = dataclasses.replace(+ ToolRefusal("#sign-in .button matches 2 elements", error_class="ambiguous_selector").as_result(),+ touched_page=True,+ )+ outcome, dispatched, _records, rounds = await _atomic_block_run(touched)++ assert dispatched == 1+ assert outcome.cap_trip == "Reached the maximum steps (1)"+ assert [[(a.succeeded, a.billable) for a in r] for r in rounds] == [[(False, True)]]+++async def test_an_atomic_block_that_is_refused_forever_ends_after_the_grace_with_its_step_charged() -> None:+ # Each refusal names a new address and reports a page change, so the repeat guard never binds;+ # the grace is the only thing that stops an uncharged refusal loop.+ clicks: list[tuple[str, dict[str, Any]]] = []++ async def handler(args: dict[str, Any]) -> ToolResult:+ clicks.append(("click", args))+ raise ToolRefusal(+ f"ref={args['n']} no longer points to an element",+ error_class="stale_ref",+ data={"page_state_changed": True},+ )++ click = ToolSpec(+ name="click", description="c", parameters={"type": "object", "properties": {}}, handler=handler, billable=True+ )+ script = [[("click", {"n": i})] for i in range(30)]+ outcome, _ = await _run(+ script, [click, make_finish_tool()], max_action_steps=1, max_action_steps_ceiling=1, max_turns=40+ )++ assert len(clicks) == UNCHARGED_REFUSAL_GRACE + 1+ assert outcome.action_steps == 1+ assert outcome.cap_trip == "Reached the maximum steps (1)"++@pytest.mark.asyncioasync def test_action_step_budget_extension_not_laundered_by_same_url_reload() -> None:# A confirmed same-URL navigate reports page_state_changed (the retry ledger legitimatelydiff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex db0ed5c74..60649ecec 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -1900,10 +1900,10 @@ async def test_a_totp_placeholder_not_typed_alone_is_refused_without_blaming_the# Nothing is wrong with the credential here: the model combined its placeholder with other text or# made one up. It is refused with words saying so, and the run is not attributed to a missing source.with _credential_totp_run(monkeypatch, seed=_TOTP_SEED) as run:- with pytest.raises(taskv3_loop.ToolRefusal) as excinfo:- await _tool(run.tools, "type").handler({"selector": "#otp", "text": text})- assert "typed alone" in str(excinfo.value)- assert "no usable" not in str(excinfo.value) and "BW_TOTP" not in str(excinfo.value)+ refused = await _tool(run.tools, "type").handler({"selector": "#otp", "text": text})+ assert refused.refused, refused.content+ assert "typed alone" in refused.content+ assert "no usable" not in refused.content and "BW_TOTP" not in refused.contentassert [c for c in run.page.calls if c[0] == "fill"] == []assert run.state.totp_source_missing is Falseassert run.state.values_delivered == 0@@ -1957,12 +1957,12 @@ async def test_a_url_argument_never_carries_a_totp_placeholder(with _credential_totp_run(monkeypatch, seed=_TOTP_SEED, other_credentials={"other_login": _OTHER_TOTP_PLACEHOLDER}) as run:- with pytest.raises(taskv3_loop.ToolRefusal) as excinfo:- await _tool(run.tools, tool_name).handler(dict(args))- assert "BW_TOTP" not in str(excinfo.value) and _TOTP_PLACEHOLDER not in str(excinfo.value)+ refused = await _tool(run.tools, tool_name).handler(dict(args))+ assert refused.refused, refused.content+ assert "BW_TOTP" not in refused.content and _TOTP_PLACEHOLDER not in refused.content# The credential's source is fine; the model sent its placeholder somewhere a code never goes.- assert "never a URL or file" in str(excinfo.value)- assert "typed alone" not in str(excinfo.value) and "no usable" not in str(excinfo.value)+ assert "never a URL or file" in refused.content+ assert "typed alone" not in refused.content and "no usable" not in refused.contentassert [c for c in run.page.calls if c[0] in ("goto", "fill", "set_input_files")] == []assert fetched == []assert run.state.values_delivered == 0@@ -9745,6 +9745,27 @@ async def test_dom_a_tab_strip_is_still_not_a_menu() -> None:assert "opened a menu of" not in r.content+ "args",+ [{"selector": '[data-tv3="t157"]'}, {"mark": 999}],+ ids=["refused_by_the_tool", "refused_by_the_address_wrapper"],+)+async def test_a_download_that_lands_during_a_refused_call_is_still_reported(+ tmp_path: Path, args: dict[str, Any]+) -> None:+ page = _ClickFakePage(exists=False)+ tools = build_browser_tools(_fixed_page_provider(page), downloads_dir=str(tmp_path))+ await _dispatch(tools, "click", dict(args))+ (tmp_path / "statement.pdf").write_bytes(b"x" * 10)++ result, refused = await _dispatch(tools, "click", dict(args))++ assert (result.status, refused) == ("error", True), result.content+ assert "Downloaded: statement.pdf (10 B)" in result.content+ assert (result.data or {}).get("download_new") is True++@pytest.mark.asyncioasync def test_click_stale_marker_fast_fails_without_15s_wait() -> None:# A [data-tv3=...] marker is minted only by our own enrichment: if it matches nothing now, it can@@ -9752,8 +9773,8 @@ async def test_click_stale_marker_fast_fails_without_15s_wait() -> None:# pure loss. Fail fast and loud, and never dispatch the doomed click.page = _ClickFakePage(exists=False)tools = build_browser_tools(_fixed_page_provider(page))- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t157"]'})- assert r.status == "error"+ r, refused = await _dispatch(tools, "click", {"selector": '[data-tv3="t157"]'})+ assert (r.status, refused) == ("error", True)assert "no longer exists" in r.contentassert "e-observe" in r.contentassert not any(c[0] == "click" for c in page.calls)@@ -9815,7 +9836,7 @@ async def test_click_on_a_marker_the_page_cloned_never_silently_lands_on_the_cloassert await page.locator(selector).count() == 2, "fixture is not armed: the clone must carry the marker"assert await page.locator(selector).first.text_content() == "Remove Beta"- r = await _tool(tools, "click").handler({"selector": selector})+ r, _ = await _dispatch(tools, "click", {"selector": selector})clicked = await page.evaluate("() => window.__clicked")if r.status == "ok":@@ -9872,9 +9893,9 @@ async def test_click_on_a_marker_the_page_destroyed_still_fails_loud() -> None:)assert await page.locator(selector).count() == 0, "fixture is not armed"- r = await _tool(tools, "click").handler({"selector": selector})+ r, refused = await _dispatch(tools, "click", {"selector": selector})- assert r.status == "error"+ assert (r.status, refused) == ("error", True)assert "no longer exists" in r.content and "e-observe" in r.contentassert r.data == {"page_state_changed": True} # a same-document re-render must poison the batchassert await page.evaluate("() => window.__clicked") == []@@ -9886,8 +9907,8 @@ async def test_click_recounts_a_marker_that_reattached_as_two_copies_during_the_# back as two copies satisfies "attached" with both present, and the count must be re-read then.page = _ClickFakePage(exists=False, match_counts=[0, 2])tools = build_browser_tools(_fixed_page_provider(page))- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t7"]'})- assert r.status == "error"+ r, refused = await _dispatch(tools, "click", {"selector": '[data-tv3="t7"]'})+ assert (r.status, refused) == ("error", True)assert "matches 2 elements" in r.content and "e-observe" in r.contentassert any(c[0] == "wait_for_selector" for c in page.calls)assert not any(c[0] == "click" for c in page.calls)@@ -9899,9 +9920,11 @@ async def test_click_timeout_on_vanished_element_reports_removal() -> None:# surfacing a generic Playwright timeout the model cannot act on.page = _ClickFakePage(exists=False, click_raises=TimeoutError("Page.click: Timeout 15000ms exceeded"))tools = build_browser_tools(_fixed_page_provider(page))- r = await _tool(tools, "click").handler({"selector": "#opt-old"})+ r, refused = await _dispatch(tools, "click", {"selector": "#opt-old"})assert r.status == "error"assert "no longer exists" in r.content+ # The click was dispatched and may have moved the page first, so it is charged, never a refusal.+ assert refused is False@pytest.mark.asyncio@@ -10357,7 +10380,7 @@ async def test_dom_stale_menu_marker_click_fails_fast_and_loud() -> None:r2 = await click.handler({"selector": "#sort-trigger"})assert "CLOSED the open menu" in r2.contentstart = time.monotonic()- r3 = await click.handler({"selector": '[data-tv3-menu="3"]'})+ r3, _ = await _dispatch([click], "click", {"selector": '[data-tv3-menu="3"]'})elapsed = time.monotonic() - startassert r3.status == "error"assert "no longer exists" in r3.content@@ -10376,7 +10399,7 @@ async def test_dom_stale_marker_click_fails_fast_and_loud() -> None:)tools = build_browser_tools(_fixed_page_provider(page))start = time.monotonic()- r = await _tool(tools, "click").handler({"selector": '[data-tv3="t9"]'})+ r, _ = await _dispatch(tools, "click", {"selector": '[data-tv3="t9"]'})elapsed = time.monotonic() - startassert r.status == "error"assert "no longer exists" in r.content@@ -10754,6 +10777,300 @@ async def test_observe_names_anonymous_shadow_hosted_controls_through_their_hostassert again_next == by_label["Next"]+async def _dispatch(tools: list[Any], name: str, args: dict[str, Any]) -> tuple[Any, bool]:+ """The result the loop records for one call, and whether the tool refused it (an uncharged call)."""+ result = await _tool(tools, name).handler(args)+ return result, result.refused+++# A rail entry and a second copy of it inside a panel. `panel_style` decides whether the copy renders.+def _two_copy_rail(panel_style: str) -> str:+ return f"""<nav id="rail"><div class="item" style="width:140px;height:24px">+ <span>Open Items</span></div></nav>+ <div id="panel" style="{panel_style}"><div class="item" style="width:140px;height:24px">+ <span>Open Items</span></div></div>+ <script>+ window.__clicked = [];+ document.querySelector('#rail .item').addEventListener('click', () => window.__clicked.push('rail'));+ document.querySelector('#panel .item').addEventListener('click', () => window.__clicked.push('panel'));+ </script>"""+++@_skip_no_browser+async def test_a_typed_selector_matching_two_is_refused_before_acting() -> None:+ async with _live_page(_two_copy_rail("")) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ for tool_name, extra in (("click", {}), ("hover", {}), ("type", {"text": "x"})):+ result, refused = await _dispatch(tools, tool_name, {"selector": "text=Open Items", **extra})+ assert (result.error_class, refused) == ("ambiguous_selector", True), (tool_name, result.content)+ assert "matches 2 elements" in result.content+ assert await page.evaluate("window.__clicked") == []+++@_skip_no_browser+async def test_a_typed_selector_that_matches_nothing_says_so_without_claiming_a_rerender() -> None:+ async with _live_page(_two_copy_rail("visibility:hidden")) as page:+ tools = build_browser_tools(_fixed_page_provider(page))++ result, refused = await _dispatch(tools, "click", {"selector": "text=Closed Items"})++ assert (result.error_class, refused) == ("stale_selector", True), result.content+ assert "matches nothing on the page" in result.content+ assert "re-rendered" not in result.content+ # The flag poisons the rest of the batch and reads as page progress; nothing moved here.+ assert not (result.data or {}).get("page_state_changed")+++# Every event a real act on the page would raise, recorded in the capture phase before any handler.+_TOUCH_RECORDER_JS = """() => {+ window.__touched = [];+ for (const type of ['pointerdown', 'mousedown', 'mouseup', 'click', 'mouseover', 'focusin', 'keydown', 'input']) {+ window.addEventListener(type, (e) => window.__touched.push(type), true);+ }+}"""+++_REFUSAL_PAGE = (+ _two_copy_rail("")+ + """<button id="save" style="width:80px;height:20px">Save</button>+ <div id="gone"><button style="width:80px;height:20px">Remove</button></div>+ <input id="code" style="width:80px;height:20px">"""+)+++async def _observed_marker(tools: list[Any], page: Any, label: str) -> str:+ await _tool(tools, "observe").handler({})+ marker = next(e["selector"] for e in (await _observe_data(page))["elements"] if e.get("label") == label)+ assert marker.startswith('[data-tv3="'), marker+ return marker+++async def _clone_remove_button(page: Any) -> None:+ await page.evaluate(+ "() => { const b = document.querySelector('#gone button'); b.parentNode.appendChild(b.cloneNode(true)); }"+ )+++_RefusalArrange = Callable[[Any, pytest.MonkeyPatch], Awaitable[tuple[list[Any], str, dict[str, Any]]]]+++async def _typed_two_visible(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ return build_browser_tools(_fixed_page_provider(page)), "click", {"selector": "text=Open Items"}+++async def _typed_zero_match(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ return build_browser_tools(_fixed_page_provider(page)), "hover", {"selector": "text=Closed Items"}+++async def _marker_vanished(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page))+ marker = await _observed_marker(tools, page, "Remove")+ await page.evaluate("() => document.querySelector('#gone').remove()")+ return tools, "hover", {"selector": marker}+++async def _marker_cloned(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page))+ marker = await _observed_marker(tools, page, "Remove")+ await _clone_remove_button(page)+ return tools, "hover", {"selector": marker}+++async def _click_marker_vanished(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools, _, args = await _marker_vanished(page, mp)+ return tools, "click", args+++async def _click_marker_cloned(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools, _, args = await _marker_cloned(page, mp)+ return tools, "click", args+++async def _mark_not_in_latest(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page))+ await _tool(tools, "look").handler({})+ return tools, "click", {"mark": 999}+++async def _stale_mark(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ await page.set_content('<button id="only" style="width:80px;height:20px">Only</button>')+ tools = build_browser_tools(_fixed_page_provider(page))+ await _tool(tools, "look").handler({})+ await page.evaluate("() => document.getElementById('only').remove()")+ return tools, "click", {"mark": 1}+++async def _invalid_mark(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ return build_browser_tools(_fixed_page_provider(page)), "type", {"mark": "first", "text": "x"}+++async def _ref_not_in_latest(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page))+ await _tool(tools, "observe").handler({})+ return tools, "type", {"selector": "ref=999", "text": "x"}+++async def _stale_ref(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page))+ ref = _ref_line((await _tool(tools, "observe").handler({})).content, "Save")+ await page.evaluate("() => document.getElementById('save').remove()")+ return tools, "click", {"selector": ref}+++async def _selector_in_two_frames(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ mp.setattr(settings, "TASK_V3_FRAME_PERCEPTION", True)+ frame = '<iframe style="width:200px;height:60px" srcdoc="<input id=pin>"></iframe>'+ await page.set_content(frame + frame)+ await page.wait_for_function(+ "() => [...document.querySelectorAll('iframe')].every(f => f.contentDocument?.getElementById('pin'))"+ )+ return build_browser_tools(_fixed_page_provider(page)), "type", {"selector": "#pin", "text": "x"}+++async def _code_inside_other_text(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page), resolve_typed_text=lambda text: text)+ return tools, "type", {"selector": "#code", "text": f"code {_TOTP_PLACEHOLDER}"}+++async def _code_with_no_resolver(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ tools = build_browser_tools(_fixed_page_provider(page), resolve_typed_text=lambda text: "BW_TOTP")+ return tools, "type", {"selector": "#code", "text": _TOTP_PLACEHOLDER}+++async def _code_with_no_secret(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ task = SimpleNamespace(task_id="tsk_otp", organization_id="o_1", workflow_run_id=None)+ tools = build_browser_tools(+ _fixed_page_provider(page),+ resolve_typed_text=lambda text: "BW_TOTP",+ resolve_totp_placeholder=VerificationState(task=task).resolve_totp_placeholder,+ )+ return tools, "type", {"selector": "#code", "text": _TOTP_PLACEHOLDER}+++async def _removed_by_the_click(page: Any, mp: pytest.MonkeyPatch) -> tuple[list[Any], str, dict[str, Any]]:+ # The pointer reaching the target removes it, so the click is dispatched and then fails as stale.+ mp.setattr(taskv3_tools, "_ACTION_TIMEOUT_MS", 1000)+ await page.set_content('<button id="flee" style="width:80px;height:20px" onmouseover="this.remove()">Flee</button>')+ return build_browser_tools(_fixed_page_provider(page)), "click", {"selector": "#flee"}+++# An async typeahead: rows appear 150ms after two typed characters, and a row click commits.+_ASYNC_TYPEAHEAD_PAGE = """<div><input id="loc" role="combobox" aria-autocomplete="list" aria-haspopup="listbox"+ aria-expanded="false" autocomplete="off" style="width:200px;height:24px"></div>+ <div id="menu" role="listbox" hidden></div>+ <div id="pair"><button class="twin" style="width:80px;height:20px">Next</button>+ <button class="twin" style="width:80px;height:20px">Next</button></div>+ <script>+ const loc = document.getElementById('loc'), menu = document.getElementById('menu');+ loc.addEventListener('input', () => {+ const q = loc.value.trim();+ setTimeout(() => {+ if (loc.value.trim() !== q || q.length < 2) return;+ menu.innerHTML = '';+ ['San Diego, California', 'San Jose, California'].filter((o) => o.includes(q)).forEach((label) => {+ const row = document.createElement('div');+ row.setAttribute('role', 'option');+ row.style.height = '30px';+ row.textContent = label;+ row.addEventListener('mousedown', (e) => e.preventDefault());+ row.addEventListener('click', () => { loc.value = label; menu.hidden = true; });+ menu.appendChild(row);+ });+ menu.hidden = false;+ loc.setAttribute('aria-expanded', 'true');+ }, 150);+ });+ </script>"""+++async def _ambiguous_after_releasing_a_live_row(+ page: Any, mp: pytest.MonkeyPatch+) -> tuple[list[Any], str, dict[str, Any]]:+ # The partial value leaves its typeahead row offered; the next call first clears that field.+ await page.set_content(_ASYNC_TYPEAHEAD_PAGE)+ tools = build_browser_tools(_fixed_page_provider(page))+ offered = await _tool(tools, "select_combobox").handler({"selector": "#loc", "value": "San Diego"})+ assert "data-tv3-pick" in offered.content, offered.content+ return tools, "click", {"selector": "#pair .twin"}+++@_skip_no_browser+ "arrange,refused",+ [+ (_typed_two_visible, True),+ (_typed_zero_match, True),+ (_marker_vanished, True),+ (_marker_cloned, True),+ (_click_marker_vanished, True),+ (_click_marker_cloned, True),+ (_mark_not_in_latest, True),+ (_stale_mark, True),+ (_invalid_mark, True),+ (_ref_not_in_latest, True),+ (_stale_ref, True),+ (_selector_in_two_frames, True),+ (_code_inside_other_text, True),+ (_code_with_no_resolver, True),+ (_code_with_no_secret, True),+ (_removed_by_the_click, False),+ (_ambiguous_after_releasing_a_live_row, False),+ ],+ ids=lambda v: v.__name__.strip("_") if callable(v) else ("refused" if v else "charged"),+)+async def test_every_uncharged_refusal_is_issued_before_the_page_is_touched(+ arrange: _RefusalArrange, refused: bool, monkeypatch: pytest.MonkeyPatch+) -> None:+ # The loop does not charge a refused call because it cannot have moved the page. Each case drives+ # one ToolRefusal raiser through the real loop; the last one is a click that dispatched and then+ # failed, which must stay charged, since it did touch the page.+ from skyvern.forge.agent import _PAGE_FINGERPRINT_PROBE_JS # noqa: PLC0415++ async with _live_page(_REFUSAL_PAGE) as page:+ tools, tool_name, args = await arrange(page, monkeypatch)+ # Parked in an empty corner and left to settle before arming: Chromium fires a synthetic mouseover+ # when layout changes under a resting pointer, which is the page's doing, not the tool's.+ await page.mouse.move(1010, 890)+ # A page that renders no frames (a background or throttled tab) never runs rAF, so the wait is bounded.+ await page.evaluate(+ "() => new Promise((r) => { requestAnimationFrame(() => requestAnimationFrame(r)); setTimeout(r, 200); })"+ )+ await page.evaluate(_TOUCH_RECORDER_JS)+ before = (page.url, await page.evaluate(_PAGE_FINGERPRINT_PROBE_JS))+ rounds: list[list[taskv3_loop.RoundAction]] = []++ async def _on_round(actions: list[taskv3_loop.RoundAction], _text: str | None) -> None:+ rounds.append(list(actions))++ outcome = await taskv3_loop.run_agent_tool_loop(+ llm_caller=_ScriptedCaller([[(tool_name, args)], [("finish", {"status": "failed", "reason": "x"})]]),+ system_prompt="sys",+ user_prompt="goal",+ tools=tools + [taskv3_loop.make_finish_tool()],+ max_turns=5,+ max_tool_calls=5,+ max_action_steps=1,+ max_action_steps_ceiling=1,+ on_action_round=_on_round,+ )++ [[action]] = rounds+ assert not action.succeeded, action+ if refused:+ assert not action.billable, action.error+ elif arrange is _removed_by_the_click:+ assert "no longer exists" in (action.error or ""), "the click did not dispatch and then go stale"+ else:+ assert "matches 2 elements" in (action.error or ""), action.error+ if not action.billable:+ assert outcome.action_steps == 0+ assert await page.evaluate("window.__touched") == [], action.error+ assert (page.url, await page.evaluate(_PAGE_FINGERPRINT_PROBE_JS)) == before, action.error++@_skip_no_browser@pytest.mark.asyncioasync def test_click_refuses_a_host_anchored_selector_the_page_has_since_cloned() -> None:@@ -10798,7 +11115,7 @@ async def test_click_refuses_a_host_anchored_selector_the_page_has_since_cloned(("select_combobox", {"value": "x"}),("hover", {}),):- cr = await _tool(tools, tool_name).handler({"selector": sel, **extra})+ cr, _ = await _dispatch(tools, tool_name, {"selector": sel, **extra})assert cr.status == "error", (tool_name, cr.content)assert "matches 2 elements" in cr.content, (tool_name, cr.content)assert await page.evaluate("window.__clicked") == []@@ -16329,7 +16646,7 @@ async def test_no_poisoned_dispatch_can_pair_a_digest_line_with_a_decoy_element(# Non-vacuous: the attack must not simply hide Alpha, or there would be nothing to mis-address.alpha_ref = _ref_line(r.content, "'Alpha'")- clicked = await _tool(tools, "click").handler({"selector": alpha_ref})+ clicked, _ = await _dispatch(tools, "click", {"selector": alpha_ref})hits = await page.evaluate("() => window.hits")assert "beta" not in hits, (attack, clicked.content, r.content)if clicked.status == "ok":@@ -16380,7 +16697,7 @@ async def test_a_ref_does_not_survive_a_navigation_onto_a_same_tag_lookalike() -await page.goto("data:text/html,<button id='go' style='width:90px;height:22px'>Elsewhere</button>")assert await page.locator("#go").count() == 1, "fixture must offer exactly one look-alike"- acted = await _tool(tools, "click").handler({"selector": ref})+ acted, _ = await _dispatch(tools, "click", {"selector": ref})assert acted.status == "error", acted.content@@ -16413,7 +16730,7 @@ async def test_a_live_handle_is_never_displaced_by_a_selector_that_drifted_onto_assert await page.locator("#go").count() == 1, "fixture must leave the selector naming ONE element"assert await page.locator("#go").first.text_content() == "Other", "and that element must be the other one"- acted = await _tool(tools, "click").handler({"selector": ref})+ acted, _ = await _dispatch(tools, "click", {"selector": ref})assert "Other" not in await page.evaluate("() => window.hits"), acted.contentassert acted.status == "error", acted.content@@ -16444,7 +16761,7 @@ async def test_a_ref_whose_element_was_rebuilt_into_twins_is_refused_rather_thanawait page.evaluate("window.__dup()")assert await page.locator("#go").count() == 2, "fixture must arm the ambiguity"- cr = await _tool(tools, "click").handler({"selector": ref})+ cr, _ = await _dispatch(tools, "click", {"selector": ref})assert cr.status == "error", cr.contentassert "re-render" in cr.content or "re-observe" in cr.content, cr.contentassert await page.evaluate("() => window.hits") == []@@ -22151,7 +22468,7 @@ async def test_a_hijacked_setattribute_cannot_hand_back_a_selector_naming_anothe" if (n === 'data-tv3-act') { return set.call(document.getElementById('decoy'), n, v); }"" return set.call(this, n, v); }; }")- result = await _tool(tools, "click").handler({"mark": 1})+ result, _ = await _dispatch(tools, "click", {"mark": 1})assert result.status == "error", result.contentassert "no longer points to an element" in result.content@@ -22182,7 +22499,7 @@ async def test_a_setattribute_trap_cannot_hand_back_a_token_two_elements_carry()" const out = Array.from(qsa.call(this, sel));"" return out.filter((e) => e.id !== 'decoy'); }; }")- result = await _tool(tools, "click").handler({"mark": 1})+ result, _ = await _dispatch(tools, "click", {"mark": 1})assert result.status == "error", result.contentassert "no longer points to an element" in result.content@@ -23426,7 +23743,7 @@ async def test_click_on_stale_suggestion_marker_fails_fast_with_reobserve_error(# as owned markers: an absent one gets the fast marker error, not a full actionability timeout.async with _content_page(_duplicate_suggestion_html(_DUPLICATE_STREET_ROWS)) as page:tools = build_browser_tools(_fixed_page_provider(page))- r = await _tool(tools, "click").handler({"selector": '[data-tv3-sugg="7"]'})+ r, _ = await _dispatch(tools, "click", {"selector": '[data-tv3-sugg="7"]'})assert r.status == "error", r.contentassert "markers vanish" in r.content, r.content@@ -31173,9 +31490,9 @@ async def test_a_ref_refuses_when_the_reading_could_not_say_which_document_it_caassert reading.status == "ok", reading.contentref = _ref_line(reading.content, "First name")- result = await _tool(tools, "click").handler({"selector": ref})+ result, refused = await _dispatch(tools, "click", {"selector": ref})- assert result.status == "error", result.content+ assert (result.status, refused) == ("error", True), result.contentassert "re-observe" in result.content.lower() or "observe()" in result.content, result.content# And it really did not act: no click reached the page for that ref.assert not any(call[0] == "click" for call in page.calls), page.calls@@ -31402,7 +31719,7 @@ async def test_every_exit_from_address_resolution_records_exactly_one_stamped_re# An address that fails to resolve still spent real time looking.readings.clear()- failed = await tools["click"].handler({"selector": "ref=1"})+ failed, _ = await _dispatch(list(tools.values()), "click", {"selector": "ref=1"})assert failed.status == "error"assert len(readings) == 1@@ -31424,7 +31741,7 @@ async def test_every_exit_from_address_resolution_records_exactly_one_stamped_re# which definition produced it, which is the one thing the contract says cannot happen.readings.clear()stamps.clear()- rejected = await tools["click"].handler({"mark": "not-an-integer"})+ rejected, _ = await _dispatch(list(tools.values()), "click", {"mark": "not-an-integer"})assert rejected.status == "error"assert len(readings) == 1assert stamps == [False]