Skip to content

Automate browser based workflows with AI

Name an icon-only close on a covering layer, and stop telling the model the layer has no controls (#8766)

pedrohsdb7d4b26accbdf
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 9475545de..d5e68ee3c 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -3203,6 +3203,8 @@ class LoopState:
# A single-action block's completion is offered through the finish tool at most once: a guard that
# holds a verdict only once would pass a second offer the model never saw it hold.
block_completion_offered: bool = False
+ # A billable action that succeeded moved the tab's URL: evidence the block's action took effect.
+ block_action_transitioned: bool = False
async def run_agent_tool_loop(
@@ -4287,23 +4289,34 @@ async def run_agent_tool_loop(
and not st.block_completion_offered
):
st.block_completion_offered = True
- block_reason = (
- f"performed the block's action ({st.billable_actions[0]}); "
- "a further action was past the block's step limit"
+ LOG.info(
+ "taskv3 block completion evidence",
+ url_changed=st.block_action_transitioned,
+ billable_actions=st.billable_actions,
)
- # Through the real handler, so every guard on a completed verdict still applies.
- try:
- block_finish = await finish_spec.handler({"status": "completed", "reason": block_reason})
- except Exception:
- LOG.warning("taskv3 block completion finish raised", exc_info=True)
- block_finish = ToolResult.error("")
- if activity is not None:
- activity.held_verdict_batch_skip = False
- if block_finish.status == "ok" and (block_finish.data or {}).get("status") == "completed":
- st.outcome = LoopOutcome("completed", block_reason)
- break
- if block_finish.status == "error":
- block_refusal = block_finish.content
+ # A successful action that moved nothing (a no-op click, a username typed before the
+ # login submits) is no sign the block is done; the finish gate's goal check, under enforce,
+ # still vetoes a URL change that landed on the wrong page.
+ if st.block_action_transitioned:
+ block_reason = (
+ f"performed the block's action ({st.billable_actions[0]}); "
+ "a further action was past the block's step limit"
+ )
+ # Through the real handler, so every guard on a completed verdict still applies.
+ try:
+ block_finish = await finish_spec.handler(
+ {"status": "completed", "reason": block_reason}
+ )
+ except Exception:
+ LOG.warning("taskv3 block completion finish raised", exc_info=True)
+ block_finish = ToolResult.error("")
+ if activity is not None:
+ activity.held_verdict_batch_skip = False
+ if block_finish.status == "ok" and (block_finish.data or {}).get("status") == "completed":
+ st.outcome = LoopOutcome("completed", block_reason)
+ break
+ if block_finish.status == "error":
+ block_refusal = block_finish.content
# Unlike the mid-batch max_tool_calls check above, the step gate is NOT special-cased
# away once the final turn is granted: a billable dispatch on the granted turn still
# hits it, which is the honest exit the grant exists to produce.
@@ -4830,6 +4843,13 @@ async def run_agent_tool_loop(
)
if spec.billable and result.status == "ok":
st.billable_actions.append(tool_name)
+ # click reports it at the top level, navigate inside its action outcome. A navigation that
+ # landed on an error page moved the URL without doing the block's action.
+ if not _outcome_reports_failure(round_outcome) and (
+ result_data.get("page_transitioned") is True
+ or (round_outcome or {}).get("page_transitioned") is True
+ ):
+ st.block_action_transitioned = True
if activity is not None and _arms_failure_evidence(tool_name, args, result.status == "ok"):
activity.last_trigger_turn = st.turns
activity.failure_evidence_trigger_generation += 1
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 4191f68f7..43d69f08e 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -346,14 +346,15 @@ def _escape_tags_in_text(text: str) -> str:
# The exact selector shapes our own enrichment mints: data-tv3 by observe(), data-tv3-menu by the
# click menu probe, data-tv3-act by act-by-mark (written on the look-resolved element at act time and
-# kept for the life of the document, so the submit watch can still resolve it turns later). Each
+# kept for the life of the document, so the submit watch can still resolve it turns later), data-tv3-close by
+# the covered probe for an unmarked onclick close. Each
# exists only where we set it, so one that matches nothing now cannot reappear without a fresh
# observe / menu-opening click / look.
_TV3_MARKER_SELECTOR_RE = re.compile(
- r'^\[data-tv3(?:(?:-menu|-act)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$'
+ r'^\[data-tv3(?:(?:-menu|-act|-close)?="[^"\\]+"\]|-sugg="[^"\\]+"\](?:\[data-tv3-pick="[^"\\]+"\])?)$'
)
# Any v3 marker inside a selector, including one observe composed a host-anchored selector around.
-_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg)?="')
+_TV3_MARKER_ANYWHERE_RE = re.compile(r'\[data-tv3(?:-menu|-act|-sugg|-close)?="')
# An opaque identifier (a uuid, or a run of 12+ hex digits) does not survive a model's copy: one
# transposed pair sends every later call to a selector that matches nothing. observe addresses its
# own elements by ref for that reason; this is what look()'s legend refuses to use as a label.
@@ -4082,6 +4083,7 @@ _LOOK_ENUM_JS = (
)
_ACT_ATTR_RE = re.compile(r'\s*data-tv3-act="[^"]*"')
+_CLOSE_ATTR_RE = re.compile(r' data-tv3-close="[^"]*"')
_ACT_SELECTOR_PREFIX = '[data-tv3-act="'
# Write the act-by-mark attribute on an element handle the caller already resolved (Playwright's
# engine, which pierces open shadow). Returns whether the node is still connected; a detached handle
@@ -4801,6 +4803,7 @@ _TYPE_TARGET_PROBE_JS = (
+ r"""
try { _q.all('[data-tv3-cover]').forEach((n) => n.removeAttribute('data-tv3-cover')); } catch (e) { /* best-effort */ }
try { _q.all('[data-tv3-catcher]').forEach((n) => n.removeAttribute('data-tv3-catcher')); } catch (e) { /* best-effort */ }
+ try { _q.all('[data-tv3-close]').forEach((n) => n.removeAttribute('data-tv3-close')); } catch (e) { /* best-effort */ }
// A host-anchored selector's two halves straddle a shadow boundary, so no single root can match it
// and a per-root lookup finds nothing -- which would read as "no field here" and skip the check on
@@ -5750,14 +5753,12 @@ _TYPE_TARGET_PROBE_JS = (
// The role list mirrors observe()'s own _WIDGET_ROLES answer to "is this a control?" (minus the
// form-field roles observe treats as fillable, not actionable), so a consent switch or a
// role=menuitem Close action is not omitted just because it isn't a <button>.
- const found = deepAll(
- layer,
- 'button,a[href],input[type="button"],input[type="submit"],input[type="image"],'
+ const controlSel = 'button,a[href],input[type="button"],input[type="submit"],input[type="image"],'
+ 'input[type="reset"],[role="button"],'
+ '[role="checkbox"],[role="radio"],[role="combobox"],[role="option"],[role="menuitem"],'
+ '[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="switch"],'
- + '[role="spinbutton"],[role="tab"]'
- );
+ + '[role="spinbutton"],[role="tab"]';
+ const found = deepAll(layer, controlSel);
// A disabled control cannot be the thing to click -- recommending one wastes a click timeout on
// a target Playwright will refuse, and can crowd the real dismisser out of the eight-slot cap.
// :disabled (not the .disabled IDL property) is what the browser actually uses to decide this,
@@ -5788,7 +5789,35 @@ _TYPE_TARGET_PROBE_JS = (
// motivating bug with more words. Keep both ends: the first few for context, the last few
// because that is where a footer actually lives.
const truncated = allControls.length > 8;
- const controls = truncated ? allControls.slice(0, 5).concat(allControls.slice(-3)) : allControls;
+ // An icon-only close is often a bare <div onclick>, which no tag or role above names. Only a leaf
+ // handler counts (a row, card or body handler wraps other controls), and at most two are added
+ // after the named ones, so a list of row handlers cannot displace the layer's own controls.
+ const leaves = [];
+ for (const c of deepAll(layer, '[onclick]').slice(0, 200)) {
+ if (c === layer || c === el || !visible(c) || isDisabled(c)) continue;
+ if (c.matches && c.matches(controlSel)) continue;
+ if (c.contains(el) || c.querySelector('[onclick],' + controlSel)) continue;
+ if (c.parentElement && c.parentElement.closest(controlSel)) continue;
+ const label = boundedClean(ownName(c) || c.textContent || (c.getAttribute && c.getAttribute('title')) || '', 60);
+ leaves.push({ c, label });
+ }
+ // An icon close has no text, so unlabeled leaves go first; a run of labelled rows cannot fill the slots ahead of it.
+ leaves.sort((a, b) => (a.label ? 1 : 0) - (b.label ? 1 : 0));
+ // Fresh per probe, so a selector an earlier probe printed matches nothing rather than another element.
+ const closeNonce = Math.random().toString(36).slice(2, 7);
+ const onclickControls = [];
+ for (const { c, label } of leaves) {
+ if (onclickControls.length >= 2) break;
+ let csel = idSelector(c) || markerSelector(c);
+ if (!csel) {
+ // A close that appeared after the last observe carries no marker, so the probe names it itself.
+ const key = closeNonce + '-' + onclickControls.length;
+ try { c.setAttribute('data-tv3-close', key); csel = '[data-tv3-close="' + key + '"]'; } catch (e) { continue; }
+ }
+ onclickControls.push({ selector: csel, label });
+ }
+ const controls = (truncated ? allControls.slice(0, 5).concat(allControls.slice(-3)) : allControls)
+ .concat(onclickControls);
try { layer.setAttribute('data-tv3-cover', '1'); } catch (e) { /* best-effort */ }
out.occluder = { selector: layerSelector, name: layerName, controls, truncated };
out.occluder.layerKind = layerKind;
@@ -12948,6 +12977,7 @@ def build_browser_tools(
# The click/type reaction gate stamps data-tv3-pre on every visible element and the reach probe
# marks the layer it names; bookkeeping that, left in place, costs truncation budget in noise.
html = html.replace(' data-tv3-pre="1"', "").replace(' data-tv3-cover="1"', "")
+ html = _CLOSE_ATTR_RE.sub("", html)
html = _COLLATERAL_ATTR_RE.sub("", html)
html = _DATE_SEGMENT_ATTR_RE.sub("", html)
# The act-by-mark tag outlives its call, so unlike the other data-tv3-* bookkeeping it is
@@ -13113,10 +13143,7 @@ def build_browser_tools(
layer_desc = f'"{name}"' if name else "a layer"
if layer_selector:
layer_desc = f"{layer_desc} ({layer_selector})"
- if parts:
- controls_desc = "; ".join(parts)
- else:
- controls_desc = "re-observe — no controls were found on it"
+ controls_desc = "; ".join(parts) if parts else "none named"
if (occluder or {}).get("truncated"):
controls_desc += "; more controls exist (re-observe to see the rest)"
if branch == "challenge":
@@ -13133,6 +13160,23 @@ def build_browser_tools(
f"{selector}.",
error_class="covered",
)
+ if not parts and (occluder or {}).get("layerKind") == "hit_fallback":
+ # The walk qualified no layer and named the element drawn at the field's position (a value cell, an
+ # option row), so there is no dialog to close.
+ return ToolResult.error(
+ f"{selector} is covered by {layer_desc}, an element drawn over it rather than a dialog, so it "
+ f"cannot be {verb}{also}. Re-observe and reach the field another way.",
+ error_class="covered",
+ )
+ if not parts:
+ # The probe names controls by tag, role and inline onclick; a close wired by a script listener is
+ # none of those, so an empty list is no evidence the layer has no way out.
+ return ToolResult.error(
+ f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. The probe found no named "
+ "controls on it, but it may still have an unlabeled close (an icon or an ×): read get_html of "
+ f"{layer_selector or 'the layer'} to find it, close the layer, then retry {selector}.",
+ error_class="covered",
+ )
return ToolResult.error(
f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. "
# The layer may be a general modal, not just a consent wall -- these are every control
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 77040d925..96e565897 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -1003,6 +1003,9 @@ async def test_spent_grant_caught_at_the_step_gate_reports_the_granting_cap() ->
assert "turn budget" in outcome.reason
+_REACHED_TARGET = {"page_transitioned": True}
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("single_action_block", [True, False], ids=["single_action", "multi_action"])
@pytest.mark.parametrize("cap", ["step_cap", "runaway"])
@@ -1013,7 +1016,11 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_
# A single-action block's contract is one action; a follow-up its step cap refused is not the block
# failing. Every other budget exit, and any block that never landed its action, stays budget_exhausted.
click_calls: list[tuple[str, dict[str, Any]]] = []
- click = (_recording_tool if acted else _erroring_tool)("click", click_calls, billable=True)
+ click = (
+ _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
+ if acted
+ else _erroring_tool("click", click_calls, billable=True)
+ )
caps: dict[str, Any] = {"max_action_steps": 1} if cap == "step_cap" else {"max_turns": 1}
script = [[("click", {})], [("click", {})], [("click", {})]]
outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=single_action_block, **caps)
@@ -1027,6 +1034,62 @@ async def test_single_action_block_completes_when_its_step_cap_refuses_a_follow_
assert outcome.status == "budget_exhausted"
[email protected]("url_changed", [False, True], ids=["url_unchanged", "url_changed"])
[email protected]("reported_by", ["click", "navigate"])
+async def test_single_action_block_completes_on_the_step_cap_only_when_its_action_moved_the_page(
+ url_changed: bool, reported_by: str
+) -> None:
+ # A click that succeeds without moving the page (a no-op control, a typed username before the login submits)
+ # is no sign the block is done. Navigate reports the move inside its action outcome.
+ data = (
+ {"page_transitioned": url_changed}
+ if reported_by == "click"
+ else {ACTION_OUTCOME_DATA_KEY: {"url": "https://example.com/b", "page_transitioned": url_changed}}
+ )
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls, billable=True, ok_data=data)
+ script = [[("click", {})], [("click", {})], [("click", {})]]
+ outcome, _ = await _run(script, [click, make_finish_tool()], single_action_block=True, max_action_steps=1)
+
+ assert outcome.status == ("completed" if url_changed else "budget_exhausted")
+ if not url_changed:
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+
+
+async def test_a_navigation_that_landed_on_an_error_page_is_no_evidence_for_the_step_cap_completion() -> None:
+ dead_end = {
+ ACTION_OUTCOME_DATA_KEY: {"url": "https://example.com/gone", "page_transitioned": True, "http_status": 404}
+ }
+ nav_calls: list[tuple[str, dict[str, Any]]] = []
+ nav = _recording_tool("navigate", nav_calls, billable=True, ok_data=dead_end)
+ script = [[("navigate", {})], [("navigate", {})], [("navigate", {})]]
+ outcome, _ = await _run(script, [nav, make_finish_tool()], single_action_block=True, max_action_steps=1)
+
+ assert outcome.status == "budget_exhausted"
+
+
[email protected]("enforce", [False, True], ids=["shadow", "enforce"])
+async def test_a_contradicting_goal_check_vetoes_the_step_cap_completion_only_under_enforce(enforce: bool) -> None:
+ # A URL change can land on the wrong page; the finish gate's goal check catches it, and its shadow arm logs
+ # without changing the outcome.
+ goal_check, verdicts = _scripted_goal_check("not_achieved")
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
+ script = [[("click", {})], [("click", {})], [("click", {})]]
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool(goal_check=goal_check, goal_check_enforce=enforce)],
+ single_action_block=True,
+ max_action_steps=1,
+ )
+
+ assert verdicts
+ assert outcome.status == ("budget_exhausted" if enforce else "completed")
+
+
@pytest.mark.asyncio
async def test_single_action_block_completion_refused_by_a_finish_guard_stays_budget_exhausted() -> None:
# The block's completion is offered through the finish tool, so a guard that would refuse the model's own
@@ -1037,7 +1100,7 @@ async def test_single_action_block_completion_refused_by_a_finish_guard_stays_bu
return refusal if status == "completed" else None
click_calls: list[tuple[str, dict[str, Any]]] = []
- click = _recording_tool("click", click_calls, billable=True)
+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
script = [[("click", {})], [("click", {})], [("click", {})]]
outcome, caller = await _run(
script,
@@ -1064,7 +1127,7 @@ async def test_single_action_block_completion_is_offered_once_even_when_its_guar
return "the submission is still pending" if len(offers) == 1 else None
click_calls: list[tuple[str, dict[str, Any]]] = []
- click = _recording_tool("click", click_calls, billable=True)
+ click = _recording_tool("click", click_calls, billable=True, ok_data=_REACHED_TARGET)
script = [[("click", {})], [("click", {})], [("click", {})]]
outcome, _ = await _run(
script,
@@ -10913,7 +10976,7 @@ async def test_every_covered_row_carries_the_branch_the_control_count_and_the_la
"""Every `covered` message logs as one `tool_error_class`, so the only way to size the split
was to pull step archives and classify the prose. The count and the layer kind are separate
facets because neither recovers the other: the named branch spans "eight controls listed" and
- "no controls were found on it", and a zero count spans a real overlay whose controls were
+ "no named controls", and a zero count spans a real overlay whose controls were
dropped and a probe that named the hit element itself, where there is nothing to dismiss."""
async def covered(args: dict[str, Any]) -> ToolResult:
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 6f4b6c23a..52986a936 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -21217,7 +21217,59 @@ async def test_covered_error_message_when_the_occluding_layer_has_no_controls_at
tools = build_browser_tools(_fixed_page_provider(page))
r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
assert r.status == "error", r.content
- assert "no controls were found on it" in r.content, r.content
+ assert "no named controls" in r.content, r.content
+
+
+_DIALOG_WHOSE_ONLY_CLOSE_IS_AN_ONCLICK_ICON_HTML = """
+<input id="city" type="text" style="width:200px;height:30px">
+<div id="trial" role="dialog" aria-label="Start your trial" style="position:fixed;left:0;top:0;width:100%;height:100%;background:#fff">
+ <div class="modal-close" onclick="this.closest('[role=dialog]').remove()" style="width:24px;height:24px;background:#000"></div>
+ <p>Unlock every feature.</p>
+</div>
+"""
+
+
+@_skip_no_browser
+async def test_an_icon_only_onclick_close_is_named_and_clicking_it_clears_the_field() -> None:
+ async with _content_page(_DIALOG_WHOSE_ONLY_CLOSE_IS_AN_ONCLICK_ICON_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
+ assert r.status == "error", r.content
+ minted = re.search(r'\[data-tv3-close="[^"]+"\]', r.content)
+ assert minted, r.content
+ closed = await _tool(tools, "click").handler({"selector": minted.group(0)})
+ assert closed.status == "ok", closed.content
+ typed = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
+ assert typed.status == "ok", typed.content
+
+
+_DIALOG_FULL_OF_ONCLICK_WRAPPERS_AND_ROWS_HTML = (
+ """
+<input id="city" type="text" style="width:200px;height:30px">
+<div id="picker" role="dialog" aria-label="Choose a plan" onclick="void 0" style="position:fixed;left:0;top:0;width:100%;height:100%;background:#fff">
+ <div id="card-a" onclick="void 0"><button id="pick-a">Pick A</button></div>
+ <div id="card-b" onclick="void 0"><button id="pick-b">Pick B</button></div>
+"""
+ + "".join(f' <div id="row-{i}" onclick="void 0">Row {i}</div>\n' for i in range(6))
+ + """ <div id="icon-close" onclick="void 0" style="width:24px;height:24px;background:#000"></div>
+ <button id="footer-close" onclick="void 0">Close</button>
+</div>
+"""
+)
+
+
+@_skip_no_browser
+async def test_onclick_wrappers_and_rows_neither_flood_the_list_nor_crowd_out_an_icon_close() -> None:
+ async with _content_page(_DIALOG_FULL_OF_ONCLICK_WRAPPERS_AND_ROWS_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
+ assert r.status == "error", r.content
+ assert r.content.count("#footer-close") == 1 and "#pick-a" in r.content, r.content
+ assert "#card-a" not in r.content and "#card-b" not in r.content, r.content
+ assert "#icon-close" in r.content, r.content
+ assert sum(f"#row-{i}" in r.content for i in range(6)) <= 1, r.content
class _CoveredProbeFakePage(_TypeaheadFakePage):
@@ -21347,7 +21399,7 @@ async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hi
tools = build_browser_tools(_fixed_page_provider(page))
r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
assert r.status == "error", r.content
- assert "no controls were found on it" in r.content, r.content
+ assert "get_html" not in r.content, r.content
recorded = taskv3_loop._COVERED_LAYER.get() or {}
assert recorded == {"branch": "named", "controls": 0, "layer_kind": "hit_fallback"}, recorded
@@ -21356,7 +21408,7 @@ async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hi
tools = build_browser_tools(_fixed_page_provider(page))
r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
assert r.status == "error", r.content
- assert "no controls were found on it" in r.content, r.content
+ assert "no named controls" in r.content, r.content
recorded = taskv3_loop._COVERED_LAYER.get() or {}
assert recorded == {"branch": "named", "controls": 0, "layer_kind": "qualified"}, recorded