Skip to content

Automate browser based workflows with AI

Stop telling Task V3 to dismiss a layer when the covering element is not one (SKY-17651) (#8787)

Andrew Neilson60c58cc8e153
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 20f079aa2..a50c9508e 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -13237,6 +13237,11 @@ def build_browser_tools(
parts = _named_controls(occluder)
branch = _covered_branch(occluder)
_record_covered(occluder, branch, controls=[] if branch == "own_container" else parts)
+ # The probe walks only the hit element and its ancestors, so this claims nothing about other layers.
+ unqualified = (
+ "That is the element drawn where the pointer lands; neither it nor anything containing it qualified "
+ "as a dialog, an overlay or a banner."
+ )
if branch == "clipped":
# A dismissal is the one instruction that cannot be carried out here: the dialog, overlay
# and banner it would name are what this branch is entered BECAUSE the probe ruled out.
@@ -13271,11 +13276,20 @@ def build_browser_tools(
layer_desc = f"a layer ({layer_selector})"
else:
layer_desc = "a layer"
- return ToolResult.error(
+ invisible = (
f"{selector} is covered by {layer_desc} that is INVISIBLE — it intercepts clicks but paints "
- f"nothing on screen, so you will not see it in a screenshot{also}. It is most likely a "
- "leftover backdrop from a dialog or cookie banner that was already dismissed. Do not keep "
- "trying to dismiss a visible overlay; press Escape, re-observe, or reach the field another way.",
+ f"nothing on screen, so you will not see it in a screenshot{also}."
+ )
+ if (occluder or {}).get("layerKind") == "hit_fallback":
+ return ToolResult.error(
+ f"{invisible} {unqualified} Re-observe and reach {selector} another way or act on a "
+ "different control.",
+ error_class="covered",
+ )
+ return ToolResult.error(
+ f"{invisible} It is most likely a leftover backdrop from a dialog or cookie banner that was "
+ "already dismissed. Do not keep trying to dismiss a visible overlay; press Escape, re-observe, "
+ "or reach the field another way.",
error_class="covered",
)
if branch == "unnamed":
@@ -13304,12 +13318,17 @@ def build_browser_tools(
f"{selector}.",
error_class="covered",
)
- if not parts and (occluder or {}).get("layerKind") == "hit_fallback":
- # The walk qualified no layer and named the element drawn at the field's position (a value cell, an
- # option row), so there is no dialog to close.
+ if (occluder or {}).get("layerKind") == "hit_fallback":
+ # The walk qualified no layer and named the element drawn at the target's position (a value cell, an
+ # option row), so any controls listed are what that element holds, not confirmed as a way to uncover the target.
+ found = (
+ f"Controls found on that element, not confirmed to uncover {selector}: {controls_desc}."
+ if parts
+ else "The probe named no controls on that element."
+ )
return ToolResult.error(
- f"{selector} is covered by {layer_desc}, an element drawn over it rather than a dialog, so it "
- f"cannot be {verb}{also}. Re-observe and reach the field another way.",
+ f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. {unqualified} "
+ f"{found} Re-observe and reach {selector} another way or act on a different control.",
error_class="covered",
)
if not parts:
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 8cf359685..91c1539d2 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -22472,31 +22472,109 @@ _COVERED_BY_AN_UNQUALIFIED_VALUE_ROW_HTML = """
<input id="city" type="text" style="position:absolute;left:0;top:0;width:200px;height:30px">
<span id="row" style="position:absolute;left:0;top:0;width:200px;height:30px;background:#fff">May</span>
"""
+_COVERED_BY_AN_UNQUALIFIED_ROW_WITH_A_CONTROL_HTML = """
+<input id="city" type="text" style="position:absolute;left:0;top:0;width:200px;height:30px">
+<div id="row" style="position:absolute;left:0;top:0;width:200px;height:30px;background:#fff">May
+ <button id="pick" style="position:absolute;right:0;top:0;width:30px;height:30px">Pick</button>
+</div>
+"""
+_COVERED_BY_AN_UNQUALIFIED_TRANSPARENT_ELEMENT_HTML = """
+<input id="city" type="text" style="position:absolute;left:0;top:0;width:200px;height:30px">
+<div id="row" style="position:absolute;left:0;top:0;width:200px;height:30px"></div>
+"""
+_UNQUALIFIED_ROW_OVER_A_DIALOG_HTML = """
+<input id="city" type="text" style="position:absolute;left:0;top:0;width:200px;height:30px">
+<div role="dialog" style="position:absolute;left:0;top:0;width:100%;height:100%;background:#eee">Notice</div>
+<div id="row" style="position:absolute;left:0;top:0;width:200px;height:30px;background:#fff">May
+ <button id="pick" style="position:absolute;right:0;top:0;width:30px;height:30px">Pick</button>
+</div>
+"""
+_COVER_REMEDY_WORDING = ("dismiss", "closes", "close the layer", "opens", "get_html", "Pick whichever")
[email protected]("short_action_timeout")
@_skip_no_browser
@pytest.mark.asyncio
-async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hit_element() -> None:
- """Both shapes render the same sentence with the same empty controls list, so the message cannot
- tell them apart -- and they are not the same event. One is an overlay whose controls the
- enumeration did not name; the other has no overlay at all."""
- async with _content_page(_COVERED_BY_AN_UNQUALIFIED_VALUE_ROW_HTML) as page:
- taskv3_loop._COVERED_LAYER.set(None)
- tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
- assert r.status == "error", r.content
- assert "get_html" not in r.content, r.content
- recorded = taskv3_loop._COVERED_LAYER.get() or {}
- assert recorded == {"branch": "named", "controls": 0, "layer_kind": "hit_fallback"}, recorded
+ ("markup", "branch", "controls", "found"),
+ [
+ (_COVERED_BY_AN_UNQUALIFIED_VALUE_ROW_HTML, "named", 0, "The probe named no controls on that element."),
+ (
+ _COVERED_BY_AN_UNQUALIFIED_ROW_WITH_A_CONTROL_HTML,
+ "named",
+ 1,
+ 'not confirmed to uncover #city: #pick "Pick".',
+ ),
+ (_COVERED_BY_AN_UNQUALIFIED_TRANSPARENT_ELEMENT_HTML, "invisible", 0, "paints nothing on screen"),
+ (_UNQUALIFIED_ROW_OVER_A_DIALOG_HTML, "named", 1, 'not confirmed to uncover #city: #pick "Pick".'),
+ ],
+ ids=["no_controls", "one_control", "invisible", "row_over_a_dialog"],
+)
+async def test_the_covered_record_separates_a_qualifying_layer_from_the_named_hit_element(
+ markup: str, branch: str, controls: int, found: str
+) -> None:
+ for verb, args in (("click", {"selector": "#city"}), ("type", {"selector": "#city", "text": "x"})):
+ async with _content_page(markup) as page:
+ taskv3_loop._COVERED_LAYER.set(None)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), verb).handler(args)
+ assert r.status == "error" and r.error_class == "covered", (verb, r.content)
+ assert "#city is covered by" in r.content and "(#row)" in r.content, (verb, r.content)
+ assert "containing it qualified" in r.content and "Nothing over it" not in r.content, (verb, r.content)
+ assert found in r.content, (verb, r.content)
+ assert "reach #city another way" in r.content, (verb, r.content)
+ assert not [w for w in _COVER_REMEDY_WORDING if w in r.content], (verb, r.content)
+ recorded = taskv3_loop._COVERED_LAYER.get() or {}
+ assert recorded == {"branch": branch, "controls": controls, "layer_kind": "hit_fallback"}, recorded
- async with _content_page(_DIALOG_WITH_NO_CONTROLS_HTML) as page:
- taskv3_loop._COVERED_LAYER.set(None)
+ async with _content_page(_DIALOG_WITH_NO_CONTROLS_HTML) as page:
+ taskv3_loop._COVERED_LAYER.set(None)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), verb).handler(args)
+ assert r.status == "error", r.content
+ assert '#city is covered by "Loading" (#wizard8)' in r.content, r.content
+ assert "no named controls" in r.content and "get_html of #wizard8" in r.content, r.content
+ recorded = taskv3_loop._COVERED_LAYER.get() or {}
+ assert recorded == {"branch": "named", "controls": 0, "layer_kind": "qualified"}, recorded
+
+
+_DISMISSIBLE_DIALOG_BESIDE_A_COVERED_ROW_HTML = """
+<input id="city" type="text" style="position:absolute;left:0;top:0;width:200px;height:30px">
+<input id="state" type="text" style="position:absolute;left:0;top:200px;width:200px;height:30px">
+<span id="row" style="position:absolute;left:0;top:200px;width:200px;height:30px;background:#fff">May</span>
+<div id="notice" role="dialog" aria-label="Cookie notice" style="position:fixed;left:0;top:0;width:100%;height:80px;background:#fff">
+ <button id="notice-close" onclick="this.closest('[role=dialog]').remove()" style="position:absolute;right:10px;top:10px">Close</button>
+</div>
+"""
+
+
[email protected]("short_action_timeout")
+@_skip_no_browser
+async def test_covered_result_preserves_real_dialog_recovery() -> None:
+ async with _content_page(_DISMISSIBLE_DIALOG_BESIDE_A_COVERED_ROW_HTML) as page:
tools = build_browser_tools(_fixed_page_provider(page))
- r = await _tool(tools, "type").handler({"selector": "#city", "text": "x"})
- assert r.status == "error", r.content
- assert "no named controls" in r.content, r.content
- recorded = taskv3_loop._COVERED_LAYER.get() or {}
- assert recorded == {"branch": "named", "controls": 0, "layer_kind": "qualified"}, recorded
+ covered = await _tool(tools, "type").handler({"selector": "#city", "text": "Iowa City"})
+ assert covered.error_class == "covered", covered.content
+ assert '"Cookie notice" (#notice)' in covered.content, covered.content
+ assert '#notice-close "Close"' in covered.content, covered.content
+ assert "Pick whichever one actually closes or dismisses the layer" in covered.content, covered.content
+
+ sibling = await _tool(tools, "type").handler({"selector": "#state", "text": "Iowa"})
+ assert sibling.error_class == "covered", sibling.content
+ assert "containing it qualified" in sibling.content, sibling.content
+ assert await page.locator("#notice").count() == 1
+
+ closed = await _tool(tools, "click").handler({"selector": "#notice-close"})
+ assert closed.status == "ok", closed.content
+ assert await page.locator("#notice").count() == 0
+
+ typed = await _tool(tools, "type").handler({"selector": "#city", "text": "Iowa City"})
+ assert typed.status == "ok", typed.content
+ assert await page.locator("#city").input_value() == "Iowa City"
+
+ again = await _tool(tools, "type").handler({"selector": "#state", "text": "Iowa"})
+ assert again.error_class == "covered", again.content
+ assert await page.locator("#state").input_value() == ""
+ assert await page.locator("#row").count() == 1
@_skip_no_browser