Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task V3 combobox collapse: node-keyed veto maps and refusal refinements (SKY-15514) (#8441)
pedrohsdb1076d3fed04a
diff --git a/pytest.ini b/pytest.iniindex 2ede9fb3e..9814b3365 100644--- a/pytest.ini+++ b/pytest.ini@@ -10,3 +10,4 @@ addopts = --capture=nonorecursedirs = tests/manual tests/evals scripts/cdp-download-poc tests/sdk eval infra/gcp-agent prompt_evaluation internal-toolsmarkers =workday_offline: opt-in offline Workday deterministic regression suite (needs a Playwright chromium; run with -m workday_offline)+ boundary: reads skyvern-frontend/ or docs/ from the repo, so it must run on FE/docs-only PRs that skip the Python suitediff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 70156ee71..55f8cccf3 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -203,7 +203,20 @@ PERCEPTION_STALL_REASON_PREFIX = "perception_stall:"# returning different content, or a download landing; a first-time probe has no baseline and is# evidence of nothing, so varied-selector probing cannot launder repetition into "progress".ACTION_LOOP_NUDGE_AFTER = 3-ACTION_LOOP_TERMINATE_AFTER = 6+# 8, not 6: RAW repeats of one action key peak at 6 across 50 completed prod runs while stuck runs+# reach 36, so 6 sat on the completed population's edge. DO NOT LOWER THIS. The effective+# post-clearing counter — what this constant is actually compared against — was then measured by+# replaying the clearing rule over per-call telemetry, and NO SEPARATION WAS OBSERVED: two completed+# runs peaked at 2 and 4, four stuck runs at 1, 3, 3 and below (upper bounds, computed identically+# on both sides), and the highest observed value fell in a COMPLETED run. n=6 is far too small to+# establish inversion as a property of either population — that spread is also consistent with+# noise — but it is no evidence FOR a lower threshold either, and lowering a safety threshold+# requires positive evidence. 8 is above everything observed, which is why the change is safe; it+# is also why nothing in the sample fires. REVISIT if the effective distribution ever becomes+# measurable at population scale; do not lower it before then. Repeat count may not be a stuck-ness+# signal at all: SKY-15602 tracks the real problem, which is that the loop has no definition of+# goal progress, only of page change.+ACTION_LOOP_TERMINATE_AFTER = 8# Facetable sibling of PERCEPTION_STALL_REASON_PREFIX; same dashboard contract.ACTION_LOOP_REASON_PREFIX = "action_loop:"@@ -1706,14 +1719,22 @@ async def run_agent_tool_loop(# (e.g. "next") on every page relies on THIS clear to survive — page_transitioned alone# deliberately does not clear the action-loop guard (see below), so only a progressed# snapshot does.- _clear_action_state()+ ring = ledger.content_only.get(action_key) if content_only_digest is not None else None+ # INVARIANT: this test and the budget-evidence test below must both refuse a return to+ # the ring. They answer one question — did the run reach ground it has not already+ # covered? — and relaxing either alone silently reopens SKY-14998: a page that CYCLES+ # moves on every probe, so `progressed` holds every round, and an unconditional clear+ # let the action DRIVING the oscillation reset its own counter forever (one production+ # key ran 11 times against a threshold of 6 while the nudge fired once).+ returned_to_known_ground = ring is not None and content_only_digest in ring+ if not returned_to_known_ground:+ _clear_action_state()# Two landed digests that differ are positive evidence, exactly like a fingerprint# mismatch — and the only movement evidence there is when page_fingerprint is absent.canonical.progress(_ProgressEvidence.PERCEPTION_DIGEST)# Evidence requires NEW content: a URL-only flip (history.pushState) still clears the# repeat guards above but earns no budget, and neither does a return to a content state# in the probe's recent ring (a panel toggling open and shut).- ring = ledger.content_only.get(action_key) if content_only_digest is not None else Noneif ring and content_only_digest not in ring:_note_page_change_evidence()if content_only_digest is not None:diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex a8b992e88..b78fdb53f 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -760,14 +760,29 @@ def _lone_duplicate_candidate(rows: list[dict[str, Any]]) -> int | None:present_vals = {str(o.get("val")) for o in rows if o.get("val") is not None}if len(present_vals) >= 2:return None- # Sets must be NESTED, not equal: an a11y copy often mirrors only a subset of the primary row's- # attributes ({"55", "New York"} beside {"55"} is one candidate). Two sets neither containing the- # other carry a genuine disagreement and refuse.- val_sets = [frozenset(str(v) for v in o.get("vals") or []) for o in rows]- non_empty_sets = [s for s in val_sets if s]- for i, a in enumerate(non_empty_sets):- for b in non_empty_sets[i + 1 :]:- if not (a <= b or b <= a):+ # Two-rule agreement over the surface+attribute-keyed entries. (1) A key present on BOTH rows+ # must carry one value — crossed values across surfaces refuse, while a value carried on one+ # row's surface only says nothing against the other row. (2) The depth-blind floor: one row's+ # flattened attr=value pairs must nest inside the other's. Rows declaring identity on disjoint+ # attributes cannot be confirmed the same, and an agreeing generic attribute (a shared title)+ # beside disjoint identifiers is ordinary markup, not identity evidence — the pairs don't nest,+ # so it refuses. A row declaring NOTHING still constrains nothing (its empty set nests, so an+ # attribute-less a11y copy collapses).+ val_maps: list[dict[str, str]] = []+ for o in rows:+ entries: dict[str, str] = {}+ for raw in o.get("vals") or []:+ key, _, val_part = str(raw).partition("=")+ entries[key] = val_part+ val_maps.append(entries)+ for i, first in enumerate(val_maps):+ for second in val_maps[i + 1 :]:+ for shared in first.keys() & second.keys():+ if first[shared] != second[shared]:+ return None+ flat_first = {f"{k.partition(':')[2]}={v}" for k, v in first.items()}+ flat_second = {f"{k.partition(':')[2]}={v}" for k, v in second.items()}+ if not (flat_first <= flat_second or flat_second <= flat_first):return Nonen = rows[0].get("n")return n if isinstance(n, int) else None@@ -873,21 +888,43 @@ def _ambiguous_rows_error()-def _row_value_suffix(o: dict[str, Any]) -> str:+def _row_value_suffix(o: dict[str, Any], rows: list[dict[str, Any]]) -> str:"""Every present distinguishing surface for one row — value, accessible label, then any otherdeclared value (data-code, data-key, ...) — each truncated to 60 chars like the row text already is,so a page-controlled attribute can never blow a refusal message up wholesale. Surfaces are additive(not first-match), since the veto may have come from a surface other than the first one present.+ `rows` is the same-text row set the refusal lists: a surface that agrees across every row+ distinguishes nothing, so the row-label clause prints only when the names disagree, and the+ capped vals render entries that differ across rows first."""parts = ""if o.get("val") is not None:parts += f" (value {str(o.get('val'))[:60]!r})"if o.get("label"):parts += f" (label {str(o.get('label'))[:60]!r})"+ # The ancestor's name is its own surface: when it differs from the preferred display label (a+ # shared leaf label masking distinct row names) AND disagrees across the rows, the disagreeing+ # name is the one that converts — gated like the veto itself, on cross-row disagreement.+ ancestor_name = (o.get("labels") or [None, None])[1]+ ancestor_names = {str((p.get("labels") or [None, None])[1]) for p in rows if (p.get("labels") or [None, None])[1]}+ if ancestor_name and str(ancestor_name) != str(o.get("label") or "") and len(ancestor_names) >= 2:+ parts += f" (row label {str(ancestor_name)[:60]!r})"# `vals` entries are attribute-keyed for comparison; render only the value part, and subtract# what the value surface already showed so a row never prints one value twice.shown_already = {str(o.get("val")).strip()} if o.get("val") is not None else set()- vals = [bare[:60] for v in (o.get("vals") or []) if (bare := str(v).split("=", 1)[-1]) not in shown_already]+ raw_vals = [str(v) for v in (o.get("vals") or [])]+ # The caller is being asked to pick between these rows: entries every row carries identically+ # cannot be what tells them apart, so the ones that differ print first (the cap must not hide+ # the distinguishing value behind agreeing generic ones).+ common_to_all = set(raw_vals)+ for p in rows:+ if p is not o:+ common_to_all &= {str(v) for v in (p.get("vals") or [])}+ vals = [+ bare[:60]+ for v in sorted(raw_vals, key=lambda v: v in common_to_all)+ if (bare := v.split("=", 1)[-1]) not in shown_already+ ]if vals:shown_vals = vals[:3]more = "; ..." if len(vals) > 3 else ""@@ -911,7 +948,7 @@ def _identical_text_rows_error(# and the stale tags may re-land on different rows at the next scan.return f'[data-tv3-sugg="{o.get("n")}"] ' if tags_live else ""- listing = "; ".join(f"{_sel(o)}{str(o.get('text') or '')[:60]!r}{_row_value_suffix(o)}" for o in shown)+ listing = "; ".join(f"{_sel(o)}{str(o.get('text') or '')[:60]!r}{_row_value_suffix(o, rows)}" for o in shown)more = len(rows) - len(shown)next_step = ('click the intended row directly by its [data-tv3-sugg="N"] selector; the typed query was left '@@ -4325,13 +4362,17 @@ _MENU_OPTION_TEXTS_JS = (// The DOM `.value` IDL is the spec submission value ALREADY -- an explicit `value=""` and an// absent attribute (which falls back to the option's own text) are genuinely distinct submission// values even though both can display identical text, so this always reads it, never null.+ // NOT trimmed: the DOM preserves whitespace in option submission values, so "x" and "x " are+ // genuinely distinct — the trim rule covers attribute-authoring drift, not submission values.val = String(el.value);} else {// Same presence rule as the OPTION branch: an attribute that EXISTS is a present value even// when empty -- `data-value=""` next to `data-value="x"` is a real disagreement, not absence.const dv = el.getAttribute('data-value');const va = el.getAttribute('value');- val = dv !== null ? dv : va;+ // data-value is authoring metadata (whitespace drift between copies is presentational); a+ // `value` attribute is a submission value and stays byte-exact like the OPTION branch above.+ val = dv !== null ? String(dv).trim() : va !== null ? String(va) : null;}// The collapse's OWN value read: the verifier's declaredValues drops all-digit and >40-char// values on purpose (they must not CONFIRM a commit), but for telling two same-text rows apart a@@ -4344,6 +4385,11 @@ _MENU_OPTION_TEXTS_JS = (const vetoVals = [];try {for (const node of new Set([el, opt || el])) {+ // Keyed by SURFACE and attribute ("a:" the option row, "l:" a leaf inside it): crossed+ // values across surfaces are disagreements a flat set cannot see. The key must come from+ // DOM structure, not tagging depth — a self-tagged row and its twin tagged at a leaf must+ // read the row's attributes under the same key, or equal renders refuse each other.+ const where = node === (opt || el) ? 'a:' : 'l:';for (const a of node.attributes) {if (!VETO_ATTR.test(a.name)) continue;// Trimmed for comparison: whitespace drift between a portal copy and an inline copy is@@ -4354,9 +4400,7 @@ _MENU_OPTION_TEXTS_JS = (// veto blind to distinct long values, while the fingerprint still tells apart any pair// differing in prefix or length. Only identical-prefix-same-length pairs read as equal.if (v.length > 512) v = v.slice(0, 512) + '#len' + v.length;- // Keyed by attribute: crossed values across surfaces (data-code="A" data-key="B" beside- // data-code="B" data-key="A") are a disagreement an unkeyed set cannot see.- vetoVals.push(a.name + '=' + v);+ vetoVals.push(where + a.name + '=' + v);}}} catch (e) { /* attributes unreadable: no veto values */ }@@ -4367,7 +4411,8 @@ _MENU_OPTION_TEXTS_JS = (setsize: Number.isFinite(setsize) && setsize > 0 ? setsize : 0,pos: pos,val: val,- vals: vetoVals.slice(0, 12),+ // 9 allowlisted attributes x 2 nodes = 18 possible entries; 24 can never truncate.+ vals: vetoVals.slice(0, 24),// Names are FULL veto inputs (truncation happens only where a message renders them): a slice// here would read two labels diverging past the cut as equal and collapse distinct rows.label: (accessibleName(el) || accessibleName(opt)) || null,@@ -5180,6 +5225,9 @@ async () => {// diverge whenever a retained control is dropped later for having no selector that names it.let hiddenKept = 0;let hiddenListed = 0;+ // Candidates the visibility gates below drop. Those drops are silent, so a page whose whole app+ // shell is behind a boot gate renders exactly like an empty one; this is what tells the two apart.+ let hiddenDropped = 0;let phantomDropped = 0;let truncated = 0;let truncatedInComponents = 0;@@ -5293,12 +5341,12 @@ async () => {// like v1 (whose center_x check is only reached for a non-zero rect), so the zero-size// skinned-proxy carve-out below still runs for an off-screen-positioned skinned control.const centerX = (gr.left + gr.width) / 2 + window.scrollX;- if (ownGated && gr.width !== 0 && gr.height !== 0 && centerX < 0 && !_hScrolledAncestor(gateEl)) { continue; }+ if (ownGated && gr.width !== 0 && gr.height !== 0 && centerX < 0 && !_hScrolledAncestor(gateEl)) { hiddenDropped++; continue; }// v1's isElementStyleVisibilityVisible (domUtils.js) drops a control whose own computed// visibility is not 'visible'. Scoped to non-zero-rect elements so the zero-size skinned-proxy// carve-out below still runs; visibility is read per-element, so a visibility:visible child of a// hidden ancestor is kept. A native checkbox/radio judges the parent here instead of itself.- if (ownGated && gr.width !== 0 && gr.height !== 0 && window.getComputedStyle(gateEl).visibility !== 'visible') { continue; }+ if (ownGated && gr.width !== 0 && gr.height !== 0 && window.getComputedStyle(gateEl).visibility !== 'visible') { hiddenDropped++; continue; }let hidden = false;if (r.width === 0 || r.height === 0) {// Design systems skin a native SELECT/checkbox/radio/file input at zero size behind a styled@@ -5319,6 +5367,7 @@ async () => {// only when it actually renders visible content, as v1's recursion does -- an empty,// all-hidden, or all-off-canvas host is a phantom.} else {+ hiddenDropped++;continue;}}@@ -5834,7 +5883,7 @@ async () => {}} catch (e) { iframeInfo.total = 0; iframeInfo.inComponents = 0; iframeInfo.entries.length = 0; iframeInfo.unread = 0; iframeInfo.failed = true; }- return JSON.stringify({ url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });+ return JSON.stringify({ url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });}""")@@ -6577,6 +6626,20 @@ def build_browser_tools(lines.append(f"note: {hidden_kept} native control(s) hidden behind styled proxies are listed with [hidden-native]")+ hidden_dropped = data.get("hiddenDropped") or 0+ # Scoped to total blindness: present-but-hidden chrome (closed menus, inactive tabs) is on+ # nearly every page, so an unconditional note would cost the prefix on every call to say+ # nothing. With no element listed the count is the whole signal -- it separates an app shell+ # still behind its boot gate from a page that genuinely has no controls, which is the one+ # distinction the model cannot otherwise make and will poll wait->observe for turns to guess.+ # Deliberately time-neutral: a boot gate resolves on its own and a stuck one never does, and+ # this cannot tell which, so the wording must not imply that waiting is what fixes it.+ if not elements and hidden_dropped:+ lines.append(+ f"note: the page has {hidden_dropped} control(s) that are present but not visible "+ "(CSS-hidden or positioned off-canvas): the DOM is populated but none of it is "+ "currently actionable"+ )phantom_dropped = data.get("phantomDropped") or 0if phantom_dropped:lines.append(@@ -6753,6 +6816,7 @@ def build_browser_tools(summary = {"text_dropped": text_dropped,"hidden_listed": hidden_kept,+ "hidden_dropped": hidden_dropped,"phantom_dropped": phantom_dropped,"iframes_in_component_roots": iframe_info.get("inComponents") or 0,"undiscovered_roots": data.get("undiscoveredRoots") or 0,@@ -8753,7 +8817,8 @@ def build_browser_tools(def _listed_row(o: dict[str, Any], canon_text: str) -> str:entry = f'[data-tv3-menu="{o.get("n")}"] {str(o.get("text") or "")[:60]!r}'if text_counts[canon_text] >= 2:- entry += _row_value_suffix(o)+ same_text = [p for p, pt in zip(shown, shown_canon_texts) if pt == canon_text]+ entry += _row_value_suffix(o, same_text)return entrylisting = "; ".join(_listed_row(o, t) for o, t in zip(shown, shown_canon_texts))diff --git a/tests/unit/test_api_docs_taxonomy.py b/tests/unit/test_api_docs_taxonomy.pyindex d834ca44a..ddab6a23c 100644--- a/tests/unit/test_api_docs_taxonomy.py+++ b/tests/unit/test_api_docs_taxonomy.py@@ -15,6 +15,10 @@ sync workflow, and `scripts/sync_openapi_docs.py --check`.import jsonfrom pathlib import Path+import pytest++pytestmark = pytest.mark.boundary+COMMITTED_SPEC = Path(__file__).resolve().parents[2] / "docs" / "api-reference" / "openapi.json"# Capitalized resource tags — kept in lockstep with .agents/skills/api-docs-audit/SKILL.md.diff --git a/tests/unit/test_docs_webhook_verification_snippets.py b/tests/unit/test_docs_webhook_verification_snippets.pyindex afd6fd31b..dda995831 100644--- a/tests/unit/test_docs_webhook_verification_snippets.py+++ b/tests/unit/test_docs_webhook_verification_snippets.py@@ -18,6 +18,8 @@ import pytestfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature+pytestmark = pytest.mark.boundary+REPO_ROOT = Path(__file__).resolve().parents[2]SNIPPET = REPO_ROOT / "docs/snippets/webhook-signature-verification.mdx"diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex c6048c8e9..6d56bb963 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -28,7 +28,9 @@ from skyvern.forge.taskv3 import loop as loop_modulefrom skyvern.forge.taskv3.loop import (ACTION_BUDGET_EXTENDED_EVENT,ACTION_BUDGET_EXTENSION_REFUSED_EVENT,+ ACTION_LOOP_NUDGE_AFTER,ACTION_LOOP_REASON_PREFIX,+ ACTION_LOOP_TERMINATE_AFTER,CANONICAL_SURVIVAL_EVENT,FAILURE_EVIDENCE_MIN_TOOL_CALLS,FAILURE_EVIDENCE_MIN_TURNS,@@ -3543,6 +3545,44 @@ async def test_action_loop_catches_varied_probe_evasion() -> None:assert caller.calls <= 15+def test_action_loop_terminate_threshold_is_pinned_to_its_measured_value() -> None:+ # The other action-loop tests read this constant so they track policy instead of drifting, which+ # leaves nothing asserting the VALUE — someone could set it to 50 and the suite would stay green.+ # Pinning it makes any change deliberate and visible in a diff, and sends the reader to the+ # do-not-lower note at the constant: the effective post-clearing counter was measured, showed no+ # separation between completed and stuck runs, and its highest observed value fell in a completed+ # run — so there is no positive evidence supporting a lower threshold.+ assert ACTION_LOOP_TERMINATE_AFTER == 8+ assert ACTION_LOOP_NUDGE_AFTER < ACTION_LOOP_TERMINATE_AFTER+++async def test_action_loop_survives_a_page_that_oscillates_between_two_known_states() -> None:+ # The production shape the guard was structurally blind to (SKY-14998, tsk in wr_568475173904014164):+ # one action key ran 11 times against a terminate threshold of 6, and the repeat nudge fired+ # exactly ONCE at repeat_count=3 before the run died on the token cap. A page that CYCLES rather+ # than freezes moves on every probe, so `snap.progressed` held every round and wiped the whole+ # repeat ledger — the action driving the oscillation reset its own counter forever. Returning to+ # a state this probe has already seen is not progress and must not clear the guard.+ from skyvern.forge.taskv3.loop import ACTION_LOOP_REASON_PREFIX++ panel = ["url=x text: 'filters panel open'", "url=x text: 'filters panel shut'"]+ contents = [panel[i % 2] for i in range(16)]+ script: list[list[tuple[str, dict[str, Any]]]] = []+ for _ in range(16):+ script.append([("click", {"selector": "#apply"})])+ script.append([("observe", {})])+ clicks: list[tuple[str, dict[str, Any]]] = []+ tools = [_billable_tool("click", clicks), _perception_tool("observe", contents), make_finish_tool()]+ outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)++ assert outcome.status == "terminated"+ assert outcome.reason.startswith(ACTION_LOOP_REASON_PREFIX)+ assert "#apply" in outcome.reason+ # Bounded well below the 16 the script offers, and below the 11 the production run reached.+ assert len(clicks) <= 10, len(clicks)++@pytest.mark.asyncioasync def test_pagination_with_changing_page_content_never_trips_action_loop() -> None:# Healthy pagination clicks the same Next selector many times, but each page's observe differs —@@ -3637,7 +3677,8 @@ async def test_action_loop_warn_and_terminate_emit_facetable_logs() -> None:warned = [entry for entry in logs if entry["event"] == "taskv3 loop action repeat nudged"]terminated = [entry for entry in logs if entry["event"] == "taskv3 loop action repeated"]assert len(warned) == 1 and warned[0]["tool"] == "click" and warned[0]["repeat_count"] == 3- assert len(terminated) == 1 and terminated[0]["tool"] == "click" and terminated[0]["repeat_count"] == 6+ assert len(terminated) == 1 and terminated[0]["tool"] == "click"+ assert terminated[0]["repeat_count"] == ACTION_LOOP_TERMINATE_AFTER@pytest.mark.asyncio@@ -4071,7 +4112,9 @@ async def test_warn_always_precedes_terminate_even_after_single_batch_burst() ->from skyvern.forge.taskv3.loop import ACTION_LOOP_REASON_PREFIXscript = [- [("click", {"selector": "#submit"})] * 5,+ # The burst must cross the terminate threshold inside ONE turn, or the property under test+ # (no verdict before a delivered warning) is never exercised.+ [("click", {"selector": "#submit"})] * ACTION_LOOP_TERMINATE_AFTER,[("click", {"selector": "#submit"})],[("click", {"selector": "#submit"})],]@@ -4080,7 +4123,8 @@ async def test_warn_always_precedes_terminate_even_after_single_batch_burst() ->outcome, _ = await _run(script, tools, max_turns=200, max_tool_calls=500)assert outcome.status == "terminated"assert outcome.reason.startswith(ACTION_LOOP_REASON_PREFIX)- assert len(clicks) == 7 # 5 burst + 1 post-warn-queue + 1 post-warn-delivery+ # burst + 1 post-warn-queue + 1 post-warn-delivery+ assert len(clicks) == ACTION_LOOP_TERMINATE_AFTER + 2warns = [m for m in outcome.messages if m.get("role") == "user" and "#submit" in str(m.get("content"))]assert len(warns) == 1assert outcome.messages.index(warns[0]) < len(outcome.messages) - 1@@ -4095,11 +4139,11 @@ async def test_action_loop_counts_errored_attempts() -> None:clicks: list[tuple[str, dict[str, Any]]] = []click = _recording_tool("click", clicks, raises=True)click.billable = True- script = [[("click", {"selector": "#dead"})] for _ in range(10)]+ script = [[("click", {"selector": "#dead"})] for _ in range(ACTION_LOOP_TERMINATE_AFTER + 4)]outcome, _ = await _run(script, [click, make_finish_tool()], max_turns=200, max_tool_calls=500)assert outcome.status == "terminated"assert outcome.reason.startswith(ACTION_LOOP_REASON_PREFIX)- assert len(clicks) == 6+ assert len(clicks) == ACTION_LOOP_TERMINATE_AFTER@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 3b43d6f06..a38e229f9 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -1679,6 +1679,7 @@ async def test_observe_result_carries_count_only_summary_for_the_call_record() -assert set(summary) == {"text_dropped","hidden_listed",+ "hidden_dropped","phantom_dropped","iframes_in_component_roots","undiscovered_roots",@@ -16226,12 +16227,15 @@ def _button_listbox_html(anchor_id: str = "cc",labels: list[str | None] | None = None,labelledby: list[str | None] | None = None,- wrap_in_span: bool = False,+ wrap_in_span: bool | list[bool] = False,span_label: str | None = None,+ span_attrs: list[dict[str, str]] | None = None,extra_attrs: list[dict[str, str]] | None = None,) -> str:labels = labels or [None] * len(rows)extra_attrs = extra_attrs or [{} for _ in rows]+ span_attrs = span_attrs or [{} for _ in rows]+ wraps = wrap_in_span if isinstance(wrap_in_span, list) else [wrap_in_span] * len(rows)# `labelledby[i]` is the TEXT of a hidden div this row's aria-labelledby points at (not an id) --# the id is minted from the row's own position so each row gets a distinct label target.labelledby = labelledby or [None] * len(rows)@@ -16248,7 +16252,9 @@ def _button_listbox_html(for k, v in extra_attrs[i].items():attrs += f" {k}={v!r}"span_attr = f" aria-label={span_label!r}" if span_label else ""- inner = f'<span style="cursor:pointer"{span_attr}>{text}</span>' if wrap_in_span else text+ for k, v in span_attrs[i].items():+ span_attr += f" {k}={v!r}"+ inner = f'<span style="cursor:pointer"{span_attr}>{text}</span>' if wraps[i] else textreturn f'<li role="option"{attrs}>{inner}</li>'li_html = "".join(@@ -16778,6 +16784,95 @@ async def test_select_combobox_refuses_rows_with_crossed_values_across_surfaces(assert r.status == "error", r.content+def test_lone_duplicate_candidate_refuses_crossed_leaf_and_ancestor_values() -> None:+ # The node dimension matters: leaf/ancestor data-code A/B beside A/A share the leaf value but+ # disagree on the ancestor's — a flat set would read the second as a subset and collapse.+ from skyvern.forge.taskv3.tools import _lone_duplicate_candidate++ rows = [+ {"n": 1, "text": "Depot", "vals": ["l:data-code=A", "a:data-code=B"]},+ {"n": 2, "text": "Depot", "vals": ["l:data-code=A", "a:data-code=A"]},+ ]+ assert _lone_duplicate_candidate(rows) is None+++def test_lone_duplicate_candidate_collapses_same_value_at_a_different_depth() -> None:+ # A copy carrying the same value on a different node is structural variance, not disagreement:+ # only keys present on BOTH rows compare.+ from skyvern.forge.taskv3.tools import _lone_duplicate_candidate++ rows = [+ {"n": 1, "text": "Depot", "vals": ["l:data-code=55"]},+ {"n": 2, "text": "Depot", "vals": ["a:data-code=55"]},+ ]+ assert _lone_duplicate_candidate(rows) == 1+++@_skip_no_browser+async def test_select_combobox_commits_duplicate_rows_with_whitespace_drifted_data_value() -> None:+ # The primary value surface trims like every other veto surface: data-value "x " beside "x" is+ # whitespace drift between copies, not a disagreement.+ rows: list[tuple[str, str | None]] = [("Depot", "x "), ("Depot", "x")]+ async with _content_page(_duplicate_suggestion_html(rows)) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})+ assert r.status == "ok", r.content+ value = await page.eval_on_selector("#addr", "el => el.value")+ assert value == "Depot", value+++@_skip_no_browser+async def test_shared_leaf_label_refusal_names_the_disagreeing_ancestor_labels() -> None:+ # When a shared leaf action label masks distinct row names, the refusal must surface the names+ # that actually disagree, or both rows print identically beside their selectors.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _button_listbox_html(rows, labels=["Branch A", "Branch B"], wrap_in_span=True, span_label="Choose")+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "error", r.content+ assert "Branch A" in r.content, r.content+ assert "Branch B" in r.content, r.content+++@_skip_no_browser+async def test_select_combobox_refuses_rows_declaring_identity_on_disjoint_attributes() -> None:+ # Two rows that each declare identity on entirely different surfaces cannot be confirmed the+ # same candidate (data-code "A" beside data-key "B" refused on main and must keep refusing).+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(_duplicate_suggestion_html(rows, attrs=[{"data-code": "A"}, {"data-key": "B"}])) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})+ assert r.status == "error", r.content+++def test_lone_duplicate_candidate_refuses_same_attribute_with_disjoint_values_across_depths() -> None:+ # The same surface naming disjoint values at different depths is a disagreement even though no+ # node+attr key is shared.+ from skyvern.forge.taskv3.tools import _lone_duplicate_candidate++ rows = [+ {"n": 1, "text": "Depot", "vals": ["l:data-code=A"]},+ {"n": 2, "text": "Depot", "vals": ["a:data-code=B"]},+ ]+ assert _lone_duplicate_candidate(rows) is None+++@_skip_no_browser+async def test_select_combobox_refuses_option_rows_with_whitespace_distinct_submission_values() -> None:+ # The DOM preserves whitespace in option submission values: "x " and "x" are distinct choices.+ rows: list[tuple[str, str | None]] = [("Foo", "x "), ("Foo", "x")]+ async with _content_page(_duplicate_suggestion_option_html(rows)) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Foo"})+ assert r.status == "error", r.content++def _grid_suggestion_html(rows: list[tuple[str, str | None]]) -> str:# The ARIA grid-combobox pattern: the tagged leaf is the gridcell, and the candidate's identity# (data-code here) lives on its [role=row] ancestor.@@ -16838,6 +16933,199 @@ async def test_select_combobox_commits_grid_duplicate_rows_sharing_the_row_code(tools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})assert r.status == "ok", r.content+++@_skip_no_browser+async def test_select_combobox_refuses_disjoint_identifiers_despite_shared_generic_attribute() -> None:+ # A generic attribute agreeing across rows (a shared tooltip) is ordinary markup, not identity+ # evidence: rows declaring identity on disjoint attributes must keep refusing even when a title+ # happens to agree on both.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _duplicate_suggestion_html(+ rows, attrs=[{"data-code": "A", "title": "Choose"}, {"data-key": "B", "title": "Choose"}]+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})+ assert r.status == "error", r.content+ committed = await page.eval_on_selector("#addr", "el => el.getAttribute('data-committed')")+ assert committed is None, committed+++def test_lone_duplicate_candidate_refuses_disjoint_identifiers_with_shared_generic_attribute() -> None:+ # Any agreeing shared attribute must not stand in for the disjoint identifiers beside it: the+ # flattened pair sets do not nest, so there is no affirmative evidence of one candidate.+ from skyvern.forge.taskv3.tools import _lone_duplicate_candidate++ rows = [+ {"n": 1, "text": "Depot", "vals": ["a:data-code=A", "a:title=Choose"]},+ {"n": 2, "text": "Depot", "vals": ["a:data-key=B", "a:title=Choose"]},+ ]+ assert _lone_duplicate_candidate(rows) is None+++def test_lone_duplicate_candidate_refuses_agreement_on_one_identifier_beside_disjoint_declarations() -> None:+ # Deliberately conservative: rows agreeing on their one shared attribute while each declaring+ # other, disjoint surfaces still refuse — a false collapse silently commits the wrong row, while+ # a refusal only hands the pick back to the caller.+ from skyvern.forge.taskv3.tools import _lone_duplicate_candidate++ rows = [+ {"n": 1, "text": "Depot", "vals": ["a:data-code=A", "a:title=T"]},+ {"n": 2, "text": "Depot", "vals": ["a:data-code=A", "l:name=N"]},+ ]+ assert _lone_duplicate_candidate(rows) is None+++@_skip_no_browser+async def test_select_combobox_commits_equal_renders_tagged_at_different_depths() -> None:+ # One candidate rendered as a wrapped leaf beside a flattened copy: the tagger lands on the span+ # in one row and on the row element itself in the other. Tagging depth is structural variance,+ # not identity — the same DOM must read the same way and collapse.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _button_listbox_html(+ rows,+ wrap_in_span=[True, False],+ extra_attrs=[{"title": "Depot, Main St"}, {"title": "Depot, Main St"}],+ span_attrs=[{"title": "Depot"}, {}],+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "ok", r.content+ value = await page.eval_on_selector("#cc-value", "el => el.value")+ assert value == "Depot", value+++@_skip_no_browser+async def test_select_combobox_refuses_leaf_values_disagreeing_under_agreeing_row_values() -> None:+ # The mirror of the crossed-depth case: leaves declare A beside B while both rows declare Z. A+ # depth-blind read would let the agreeing row value shadow the leaf disagreement and collapse+ # two distinct rows.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _button_listbox_html(+ rows,+ wrap_in_span=True,+ extra_attrs=[{"data-code": "Z"}, {"data-code": "Z"}],+ span_attrs=[{"data-code": "A"}, {"data-code": "B"}],+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "error", r.content+ value = await page.eval_on_selector("#cc-value", "el => el.value")+ assert value == "", value+++@_skip_no_browser+async def test_select_combobox_commits_duplicate_rows_carrying_the_same_attribute_at_both_depths() -> None:+ # The accept direction of the two-depth read: twin renders agreeing at the leaf AND the row.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _button_listbox_html(+ rows,+ wrap_in_span=True,+ extra_attrs=[{"data-code": "Z"}, {"data-code": "Z"}],+ span_attrs=[{"data-code": "A"}, {"data-code": "A"}],+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "ok", r.content+ value = await page.eval_on_selector("#cc-value", "el => el.value")+ assert value == "Depot", value+++@_skip_no_browser+async def test_select_combobox_refuses_rows_whose_distinguishing_value_sits_past_twelve_entries() -> None:+ # The value list must never truncate ahead of the disagreeing entry: nine allowlisted leaf+ # attributes plus the row's own push the rows' disagreeing data-code to position 13, which a+ # 12-entry cap discarded — both rows then read identical and the first was committed.+ nine = {+ "value": "v",+ "data-value": "v",+ "data-val": "v",+ "data-v": "v",+ "data-code": "C",+ "data-key": "k",+ "data-option-value": "o",+ "name": "n",+ "title": "t",+ }+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _button_listbox_html(+ rows,+ wrap_in_span=True,+ span_attrs=[dict(nine), dict(nine)],+ extra_attrs=[+ {"title": "t", "name": "n", "value": "v", "data-code": "ROW-A"},+ {"title": "t", "name": "n", "value": "v", "data-code": "ROW-B"},+ ],+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "error", r.content+ value = await page.eval_on_selector("#cc-value", "el => el.value")+ assert value == "", value+++@_skip_no_browser+async def test_select_combobox_refuses_rows_with_whitespace_distinct_value_attribute() -> None:+ # A `value` attribute is a submission value like OPTION.value, not authoring metadata: "x " and+ # "x" are two choices, so byte-exact comparison covers it and only data-value gets the drift trim.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(_duplicate_suggestion_html(rows, attrs=[{"value": "x "}, {"value": "x"}])) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})+ assert r.status == "error", r.content+++@_skip_no_browser+async def test_identical_text_refusal_omits_row_labels_that_agree_across_rows() -> None:+ # The row-label clause exists to surface the surface that DISAGREES; when every row wears the+ # same ancestor name the clause distinguishes nothing and must not print at all.+ rows: list[tuple[str, str | None]] = [("Depot", "v1"), ("Depot", "v2")]+ async with _content_page(+ _button_listbox_html(rows, labels=["Same Branch", "Same Branch"], wrap_in_span=True, span_label="Choose")+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#cc", "value": "Depot"})+ assert r.status == "error", r.content+ assert "row label" not in r.content, r.content+++@_skip_no_browser+async def test_identical_text_refusal_leads_with_the_value_that_disagrees() -> None:+ # Agreeing generic values must not crowd the disagreeing identity out of the capped suffix —+ # the caller is being asked to pick between the rows, so what differs prints first.+ rows: list[tuple[str, str | None]] = [("Depot", None), ("Depot", None)]+ async with _content_page(+ _duplicate_suggestion_html(+ rows,+ attrs=[+ {"title": "T", "name": "N", "data-val": "V", "data-code": "AAA"},+ {"title": "T", "name": "N", "data-val": "V", "data-code": "BBB"},+ ],+ )+ ) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "select_combobox").handler({"selector": "#addr", "value": "Depot"})+ assert r.status == "error", r.content+ assert "AAA" in r.content, r.content+ assert "BBB" in r.content, r.contentvalue = await page.eval_on_selector("#addr", "el => el.value")assert value == "Depot", value@@ -20849,3 +21137,58 @@ async def test_semantic_verify_kill_switch_keeps_commits_working(monkeypatch: py{"selector": "#city", "value": "Springfield, Sangamon, IL"})assert picked.status == "ok", picked.content+++@_skip_no_browser+async def test_observe_discloses_that_a_populated_page_was_dropped_whole_by_the_visibility_gates() -> None:+ # An app shell that keeps its content in the DOM behind a boot gate renders `(0 interactive+ # elements)` -- byte-identical to a genuinely empty page, because the visibility gates drop+ # silently. That ambiguity is what makes a model poll wait->observe for turns on end: nothing in+ # the payload says the page is full and unreadable rather than still loading. v1 is equally blind+ # here (measured), so the gates stay; only the disclosure is new.+ html = (+ "<!doctype html><html><head><title>Portal</title>"+ "<style>#shell { visibility: hidden; }</style></head><body>"+ '<div id="shell">'+ '<nav><a href="#/home">Home</a><a href="#/logout">Logout</a></nav>'+ "<h1>Document Portal</h1><p>Welcome back.</p>"+ '<button id="newdoc">New Document</button>'+ '<input id="q" type="text" placeholder="Search documents">'+ '<div role="status">Signed in successfully</div>'+ "</div>"+ # Outside the hidden shell, so it is the off-canvas gate that drops this one and not the+ # visibility gate: the count then discriminates both gates rather than only the second.+ '<button id="ghost" style="position:absolute;left:-9999px">Ghost</button>'+ "</body></html>"+ )+ async with _content_page(html) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "observe").handler({})++ assert r.status == "ok", r.content+ assert "(0 interactive elements)" in r.content, r.content+ assert "note: the page has 5 control(s) that are present but not visible" in r.content, r.content+ assert r.data is not None and r.data["summary"]["hidden_dropped"] == 5+++@_skip_no_browser+async def test_observe_stays_silent_about_hidden_chrome_when_it_can_still_see_the_page() -> None:+ # The note is scoped to total blindness on purpose: site chrome is routinely present-but-hidden+ # (closed menus, inactive tabs), so a note on every observe would be noise on the one channel+ # whose cost is ~linear in the prefix. A page the model can act on says nothing.+ html = (+ "<!doctype html><html><head><title>Portal</title></head><body>"+ '<button id="go">Go</button>'+ '<div style="display:none"><button id="hidden-menu">Archive</button></div>'+ "</body></html>"+ )+ async with _content_page(html) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "observe").handler({})++ assert r.status == "ok", r.content+ assert "(1 interactive elements)" in r.content, r.content+ assert "present but not visible" not in r.content, r.content+ assert r.data is not None and r.data["summary"]["hidden_dropped"] == 1