Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Name an icon-only close on a covering layer, and stop telling the model the layer has no controls (#8766)
pedrohsdb7d4b26accbdf
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 9475545de..d5e68ee3c 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -3203,6 +3203,8 @@ class LoopState:# A single-action block's completion is offered through the finish tool at most once: a guard that# holds a verdict only once would pass a second offer the model never saw it hold.block_completion_offered: bool = False+ # A billable action that succeeded moved the tab's URL: evidence the block's action took effect.+ block_action_transitioned: bool = Falseasync def run_agent_tool_loop(@@ -4287,23 +4289,34 @@ async def run_agent_tool_loop(and not st.block_completion_offered):st.block_completion_offered = True- block_reason = (- f"performed the block's action ({st.billable_actions[0]}); "- "a further action was past the block's step limit"+ LOG.info(+ "taskv3 block completion evidence",+ url_changed=st.block_action_transitioned,+ billable_actions=st.billable_actions,)- # Through the real handler, so every guard on a completed verdict still applies.- try:- block_finish = await finish_spec.handler({"status": "completed", "reason": block_reason})- except Exception:- LOG.warning("taskv3 block completion finish raised", exc_info=True)- block_finish = ToolResult.error("")- if activity is not None:- activity.held_verdict_batch_skip = False- if block_finish.status == "ok" and (block_finish.data or {}).get("status") == "completed":- st.outcome = LoopOutcome("completed", block_reason)- break- if block_finish.status == "error":- block_refusal = block_finish.content+ # A successful action that moved nothing (a no-op click, a username typed before the+ # login submits) is no sign the block is done; the finish gate's goal check, under enforce,+ # still vetoes a URL change that landed on the wrong page.+ if st.block_action_transitioned:+ block_reason = (+ f"performed the block's action ({st.billable_actions[0]}); "+ "a further action was past the block's step limit"+ )+ # Through the real handler, so every guard on a completed verdict still applies.+ try:+ block_finish = await finish_spec.handler(+ {"status": "completed", "reason": block_reason}+ )+ except Exception:+ LOG.warning("taskv3 block completion finish raised", exc_info=True)+ block_finish = ToolResult.error("")+ if activity is not None:+ activity.held_verdict_batch_skip = False+ if block_finish.status == "ok" and (block_finish.data or {}).get("status") == "completed":+ st.outcome = LoopOutcome("completed", block_reason)+ break+ if block_finish.status == "error":+ block_refusal = block_finish.content# Unlike the mid-batch max_tool_calls check above, the step gate is NOT special-cased# away once the final turn is granted: a billable dispatch on the granted turn still# hits it, which is the honest exit the grant exists to produce.@@ -4830,6 +4843,13 @@ async def run_agent_tool_loop()if spec.billable and result.status == "ok":st.billable_actions.append(tool_name)+ # click reports it at the top level, navigate inside its action outcome. A navigation that+ # landed on an error page moved the URL without doing the block's action.+ if not _outcome_reports_failure(round_outcome) and (+ result_data.get("page_transitioned") is True+ or (round_outcome or {}).get("page_transitioned") is True+ ):+ st.block_action_transitioned = Trueif activity is not None and _arms_failure_evidence(tool_name, args, result.status == "ok"):activity.last_trigger_turn = st.turnsactivity.failure_evidence_trigger_generation += 1diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 4191f68f7..43d69f08e 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -346,14 +346,15 @@ def _escape_tags_in_text(text: str) -> str:# The exact selector shapes our own enrichment mints: data-tv3 by observe(), data-tv3-menu by the# click menu probe, data-tv3-act by act-by-mark (written on the look-resolved element at act time and-# kept for the life of the document, so the submit watch can still resolve it turns later). Each+# kept for the life of the document, so the submit watch can still resolve it turns later), data-tv3-close by+# the covered probe for an unmarked onclick close. Each# exists only where we set it, so one that matches nothing now cannot reappear without a fresh# observe / menu-opening click / look._TV3_MARKER_SELECTOR_RE = re.compile(- r'^\[data-tv3(?:(?:-menu|-act)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$'+ r'^\[data-tv3(?:(?:-menu|-act|-close)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$')# Any v3 marker inside a selector, including one observe composed a host-anchored selector around.-_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg)?="')+_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg|-close)?="')# An opaque identifier (a uuid, or a run of 12+ hex digits) does not survive a model's copy: one# transposed pair sends every later call to a selector that matches nothing. observe addresses its# own elements by ref for that reason; this is what look()'s legend refuses to use as a label.@@ -4082,6 +4083,7 @@ _LOOK_ENUM_JS = ()_ACT_ATTR_RE = re.compile(r'\s*data-tv3-act="[^"]*"')+_CLOSE_ATTR_RE = re.compile(r' data-tv3-close="[^"]*"')_ACT_SELECTOR_PREFIX = '[data-tv3-act="'# Write the act-by-mark attribute on an element handle the caller already resolved (Playwright's# engine, which pierces open shadow). Returns whether the node is still connected; a detached handle@@ -4801,6 +4803,7 @@ _TYPE_TARGET_PROBE_JS = (+ r"""try { _q.all('[data-tv3-cover]').forEach((n) => n.removeAttribute('data-tv3-cover')); } catch (e) { /* best-effort */ }try { _q.all('[data-tv3-catcher]').forEach((n) => n.removeAttribute('data-tv3-catcher')); } catch (e) { /* best-effort */ }+ try { _q.all('[data-tv3-close]').forEach((n) => n.removeAttribute('data-tv3-close')); } catch (e) { /* best-effort */ }// A host-anchored selector's two halves straddle a shadow boundary, so no single root can match it// and a per-root lookup finds nothing -- which would read as "no field here" and skip the check on@@ -5750,14 +5753,12 @@ _TYPE_TARGET_PROBE_JS = (// The role list mirrors observe()'s own _WIDGET_ROLES answer to "is this a control?" (minus the// form-field roles observe treats as fillable, not actionable), so a consent switch or a// role=menuitem Close action is not omitted just because it isn't a <button>.- const found = deepAll(- layer,- 'button,a[href],input[type="button"],input[type="submit"],input[type="image"],'+ const controlSel = 'button,a[href],input[type="button"],input[type="submit"],input[type="image"],'+ 'input[type="reset"],[role="button"],'+ '[role="checkbox"],[role="radio"],[role="combobox"],[role="option"],[role="menuitem"],'+ '[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="switch"],'- + '[role="spinbutton"],[role="tab"]'- );+ + '[role="spinbutton"],[role="tab"]';+ const found = deepAll(layer, controlSel);// A disabled control cannot be the thing to click -- recommending one wastes a click timeout on// a target Playwright will refuse, and can crowd the real dismisser out of the eight-slot cap.// :disabled (not the .disabled IDL property) is what the browser actually uses to decide this,@@ -5788,7 +5789,35 @@ _TYPE_TARGET_PROBE_JS = (// motivating bug with more words. Keep both ends: the first few for context, the last few// because that is where a footer actually lives.const truncated = allControls.length > 8;- const controls = truncated ? allControls.slice(0, 5).concat(allControls.slice(-3)) : allControls;+ // An icon-only close is often a bare <div onclick>, which no tag or role above names. Only a leaf+ // handler counts (a row, card or body handler wraps other controls), and at most two are added+ // after the named ones, so a list of row handlers cannot displace the layer's own controls.+ const leaves = [];+ for (const c of deepAll(layer, '[onclick]').slice(0, 200)) {+ if (c === layer || c === el || !visible(c) || isDisabled(c)) continue;+ if (c.matches && c.matches(controlSel)) continue;+ if (c.contains(el) || c.querySelector('[onclick],' + controlSel)) continue;+ if (c.parentElement && c.parentElement.closest(controlSel)) continue;+ const label = boundedClean(ownName(c) || c.textContent || (c.getAttribute && c.getAttribute('title')) || '', 60);+ leaves.push({ c, label });+ }+ // An icon close has no text, so unlabeled leaves go first; a run of labelled rows cannot fill the slots ahead of it.+ leaves.sort((a, b) => (a.label ? 1 : 0) - (b.label ? 1 : 0));+ // Fresh per probe, so a selector an earlier probe printed matches nothing rather than another element.+ const closeNonce = Math.random().toString(36).slice(2, 7);+ const onclickControls = [];+ for (const { c, label } of leaves) {+ if (onclickControls.length >= 2) break;+ let csel = idSelector(c) || markerSelector(c);+ if (!csel) {+ // A close that appeared after the last observe carries no marker, so the probe names it itself.+ const key = closeNonce + '-' + onclickControls.length;+ try { c.setAttribute('data-tv3-close', key); csel = '[data-tv3-close="' + key + '"]'; } catch (e) { continue; }+ }+ onclickControls.push({ selector: csel, label });+ }+ const controls = (truncated ? allControls.slice(0, 5).concat(allControls.slice(-3)) : allControls)+ .concat(onclickControls);try { layer.setAttribute('data-tv3-cover', '1'); } catch (e) { /* best-effort */ }out.occluder = { selector: layerSelector, name: layerName, controls, truncated };out.occluder.layerKind = layerKind;@@ -12948,6 +12977,7 @@ def build_browser_tools(# The click/type reaction gate stamps data-tv3-pre on every visible element and the reach probe# marks the layer it names; bookkeeping that, left in place, costs truncation budget in noise.html = html.replace(' data-tv3-pre="1"', "").replace(' data-tv3-cover="1"', "")+ html = _CLOSE_ATTR_RE.sub("", html)html = _COLLATERAL_ATTR_RE.sub("", html)html = _DATE_SEGMENT_ATTR_RE.sub("", html)# The act-by-mark tag outlives its call, so unlike the other data-tv3-* bookkeeping it is@@ -13113,10 +13143,7 @@ def build_browser_tools(layer_desc = f'"{name}"' if name else "a layer"if layer_selector:layer_desc = f"{layer_desc} ({layer_selector})"- if parts:- controls_desc = "; ".join(parts)- else:- controls_desc = "re-observe — no controls were found on it"+ controls_desc = "; ".join(parts) if parts else "none named"if (occluder or {}).get("truncated"):controls_desc += "; more controls exist (re-observe to see the rest)"if branch == "challenge":@@ -13133,6 +13160,23 @@ def build_browser_tools(f"{selector}.",error_class="covered",)+ if not parts and (occluder or {}).get("layerKind") == "hit_fallback":+ # The walk qualified no layer and named the element drawn at the field's position (a value cell, an+ # option row), so there is no dialog to close.+ return ToolResult.error(+ f"{selector} is covered by {layer_desc}, an element drawn over it rather than a dialog, so it "+ f"cannot be {verb}{also}. Re-observe and reach the field another way.",+ error_class="covered",+ )+ if not parts:+ # The probe names controls by tag, role and inline onclick; a close wired by a script listener is+ # none of those, so an empty list is no evidence the layer has no way out.+ return ToolResult.error(+ f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. The probe found no named "+ "controls on it, but it may still have an unlabeled close (an icon or an ×): read get_html of "+ f"{layer_selector or 'the layer'} to find it, close the layer, then retry {selector}.",+ error_class="covered",+ )return ToolResult.error(f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. "# The layer may be a general modal, not just a consent wall -- these are every controldiff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 77040d925..96e565897 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -1003,6 +1003,9 @@ async def test_spent_grant_caught_at_the_step_gate_reports_the_granting_cap() ->assert "turn budget" in outcome.reason+_REACHED_TARGET = {"page_transitioned": True}++@pytest.mark.asyncio@pytest.mark.parametrize("single_action_block", [True, False], ids=["single_action", "multi_action"])@pytest.mark.parametrize("cap", ["step_cap", "runaway"])@@ -1013,7 +1016,11 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_# A single-action block's contract is one action; a follow-up its step cap refused is not the block# failing. Every other budget exit, and any block that never landed its action, stays budget_exhausted.click_calls: list[tuple[str, dict[str, Any]]] = []- click = (_recording_tool if acted else _erroring_tool)("click", click_calls, billable=True)+ click = (+ _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)+ if acted+ else _erroring_tool("click", click_calls, billable=True)+ )caps: dict[str, Any] = {"max_action_steps": 1} if cap == "step_cap" else {"max_turns": 1}script = [[("click", {})], [("click", {})], [("click", {})]]outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=single_action_block, **caps)@@ -1027,6 +1034,62 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_assert outcome.status == "budget_exhausted"[email protected]("url_changed", [False, True], ids=["url_unchanged", "url_changed"])[email protected]("reported_by", ["click", "navigate"])+async def test_single_action_block_completes_on_the_step_cap_only_when_its_action_moved_the_page(+ url_changed: bool, reported_by: str+) -> None:+ # A click that succeeds without moving the page (a no-op control, a typed username before the login submits)+ # is no sign the block is done. Navigate reports the move inside its action outcome.+ data = (+ {"page_transitioned": url_changed}+ if reported_by == "click"+ else {ACTION_OUTCOME_DATA_KEY: {"url": "https://example.com/b", "page_transitioned": url_changed}}+ )+ click_calls: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", click_calls, billable=True, ok_data=data)+ script = [[("click", {})], [("click", {})], [("click", {})]]+ outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=True, max_action_steps=1)++ assert outcome.status == ("completed" if url_changed else "budget_exhausted")+ if not url_changed:+ assert outcome.cap_trip == "Reached the maximum steps (1)"+++async def test_a_navigation_that_landed_on_an_error_page_is_no_evidence_for_the_step_cap_completion() -> None:+ dead_end = {+ ACTION_OUTCOME_DATA_KEY: {"url": "https://example.com/gone", "page_transitioned": True, "http_status": 404}+ }+ nav_calls: list[tuple[str, dict[str, Any]]] = []+ nav = _recording_tool("navigate", nav_calls, billable=True, ok_data=dead_end)+ script = [[("navigate", {})], [("navigate", {})], [("navigate", {})]]+ outcome, _ = await _run(script, [nav, make_finish_tool()], single_action_block=True, max_action_steps=1)++ assert outcome.status == "budget_exhausted"++[email protected]("enforce", [False, True], ids=["shadow", "enforce"])+async def test_a_contradicting_goal_check_vetoes_the_step_cap_completion_only_under_enforce(enforce: bool) -> None:+ # A URL change can land on the wrong page; the finish gate's goal check catches it, and its shadow arm logs+ # without changing the outcome.+ goal_check, verdicts = _scripted_goal_check("not_achieved")+ click_calls: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)+ script = [[("click", {})], [("click", {})], [("click", {})]]+ outcome, _ = await _run(+ script,+ [click, make_finish_tool(goal_check=goal_check, goal_check_enforce=enforce)],+ single_action_block=True,+ max_action_steps=1,+ )++ assert verdicts+ assert outcome.status == ("budget_exhausted" if enforce else "completed")++@pytest.mark.asyncioasync def test_single_action_block_completion_refused_by_a_finish_guard_stays_budget_exhausted() -> None:# The block's completion is offered through the finish tool, so a guard that would refuse the model's own@@ -1037,7 +1100,7 @@ async def test_single_action_block_completion_refused_by_a_finish_guard_stays_bureturn refusal if status == "completed" else Noneclick_calls: list[tuple[str, dict[str, Any]]] = []- click = _recording_tool("click", click_calls, billable=True)+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)script = [[("click", {})], [("click", {})], [("click", {})]]outcome, caller = await _run(script,@@ -1064,7 +1127,7 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guarreturn "the submission is still pending" if len(offers) == 1 else Noneclick_calls: list[tuple[str, dict[str, Any]]] = []- click = _recording_tool("click", click_calls, billable=True)+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)script = [[("click", {})], [("click", {})], [("click", {})]]outcome, _ = await _run(script,@@ -10913,7 +10976,7 @@ async def test_every_covered_row_carries_the_branch_the_control_count_and_the_la"""Every `covered` message logs as one `tool_error_class`, so the only way to size the splitwas to pull step archives and classify the prose. The count and the layer kind are separatefacets because neither recovers the other: the named branch spans "eight controls listed" and- "no controls were found on it", and a zero count spans a real overlay whose controls were+ "no named controls", and a zero count spans a real overlay whose controls weredropped and a probe that named the hit element itself, where there is nothing to dismiss."""async def covered(args: dict[str, Any]) -> ToolResult:diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 6f4b6c23a..52986a936 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -21217,7 +21217,59 @@ async def test_covered_error_message_when_the_occluding_layer_has_no_controls_attools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})assert r.status == "error", r.content- assert "no controls were found on it" in r.content, r.content+ assert "no named controls" in r.content, r.content+++_DIALOG_WHOSE_ONLY_CLOSE_IS_AN_ONCLICK_ICON_HTML = """+<input id="city" type="text" style="width:200px;height:30px">+<div id="trial" role="dialog" aria-label="Start your trial" style="position:fixed;left:0;top:0;width:100%;height:100%;background:#fff">+ <div class="modal-close" onclick="this.closest('[role=dialog]').remove()" style="width:24px;height:24px;background:#000"></div>+ <p>Unlock every feature.</p>+</div>+"""+++@_skip_no_browser+async def test_an_icon_only_onclick_close_is_named_and_clicking_it_clears_the_field() -> None:+ async with _content_page(_DIALOG_WHOSE_ONLY_CLOSE_IS_AN_ONCLICK_ICON_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})+ assert r.status == "error", r.content+ minted = re.search(r'\[data-tv3-close="[^"]+"\]', r.content)+ assert minted, r.content+ closed = await _tool(tools, "click").handler({"selector": minted.group(0)})+ assert closed.status == "ok", closed.content+ typed = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})+ assert typed.status == "ok", typed.content+++_DIALOG_FULL_OF_ONCLICK_WRAPPERS_AND_ROWS_HTML = (+ """+<input id="city" type="text" style="width:200px;height:30px">+<div id="picker" role="dialog" aria-label="Choose a plan" onclick="void 0" style="position:fixed;left:0;top:0;width:100%;height:100%;background:#fff">+ <div id="card-a" onclick="void 0"><button id="pick-a">Pick A</button></div>+ <div id="card-b" onclick="void 0"><button id="pick-b">Pick B</button></div>+"""+ + "".join(f' <div id="row-{i}" onclick="void 0">Row {i}</div>\n' for i in range(6))+ + """ <div id="icon-close" onclick="void 0" style="width:24px;height:24px;background:#000"></div>+ <button id="footer-close" onclick="void 0">Close</button>+</div>+"""+)+++@_skip_no_browser+async def test_onclick_wrappers_and_rows_neither_flood_the_list_nor_crowd_out_an_icon_close() -> None:+ async with _content_page(_DIALOG_FULL_OF_ONCLICK_WRAPPERS_AND_ROWS_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})+ assert r.status == "error", r.content+ assert r.content.count("#footer-close") == 1 and "#pick-a" in r.content, r.content+ assert "#card-a" not in r.content and "#card-b" not in r.content, r.content+ assert "#icon-close" in r.content, r.content+ assert sum(f"#row-{i}" in r.content for i in range(6)) <= 1, r.contentclass _CoveredProbeFakePage(_TypeaheadFakePage):@@ -21347,7 +21399,7 @@ async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hitools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})assert r.status == "error", r.content- assert "no controls were found on it" in r.content, r.content+ assert "get_html" not in r.content, r.contentrecorded = taskv3_loop._COVERED_LAYER.get() or {}assert recorded == {"branch": "named", "controls": 0, "layer_kind": "hit_fallback"}, recorded@@ -21356,7 +21408,7 @@ async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hitools = build_browser_tools(_fixed_page_provider(page))r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})assert r.status == "error", r.content- assert "no controls were found on it" in r.content, r.content+ assert "no named controls" in r.content, r.contentrecorded = taskv3_loop._COVERED_LAYER.get() or {}assert recorded == {"branch": "named", "controls": 0, "layer_kind": "qualified"}, recorded