Skip to content

Automate browser based workflows with AI

Click a transparent checkbox laid over its own styled box in Task V3 instead of calling it unrendered (#8734)

pedrohsdb44f471fe9ee3
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 9bcc97d5c..d7a97cc1c 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -5906,9 +5906,65 @@ _SKINNED_CHECKBOX_PROBE_JS = (
}
if (!invisible) return { exists: true, skinned: false, labelClick: null, ...toggleFields(_toggleOwner(el)) };
const radio = type === 'radio';
- if (!_nativeProxy(el)) return { exists: true, skinned: false, labelClick: null, radio, unproxied: true };
+ // The click lands on coordinates this realm measured, never through a marker the page could move
+ // onto something else, so the point has to be the target's own: a cover there is a cover.
+ // Returns the point on a hit on the target, false under an unrelated element, null for no hit or a clipped-off point.
+ // A clip that can scroll (a long list) is scrolled once to bring the target into it; only one that cannot is final.
+ const centreHit = (t, retried) => {
+ const mid = () => { const b = t.getBoundingClientRect(); return [b.left + b.width / 2, b.top + b.height / 2]; };
+ let [x, y] = mid();
+ if (x < 0 || y < 0 || x >= innerWidth || y >= innerHeight) {
+ try { t.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { /* keep */ }
+ [x, y] = mid();
+ }
+ let hit = null;
+ try { hit = document.elementFromPoint(x, y); } catch (e) { hit = null; }
+ for (let hops = 0; hit && hops < 32; hops++) {
+ let root = null;
+ try { root = hit.shadowRoot; } catch (e) { break; }
+ if (!root || root.nodeType !== 11) break;
+ let inner = null;
+ try { inner = root.elementFromPoint(x, y); } catch (e) { break; }
+ if (!inner || inner === hit) break;
+ hit = inner;
+ }
+ for (let n = hit, hops = 0; n && hops < 256; hops++, n = n.assignedSlot || n.parentNode || n.host || null) {
+ if (n === t || n === el) return { x, y };
+ }
+ if (!hit) return null;
+ for (let n = t.assignedSlot || t.parentNode || t.host, hops = 0; n && hops < 256; hops++, n = n.assignedSlot || n.parentNode || n.host || null) {
+ if (n.nodeType !== 1) continue;
+ const o = getComputedStyle(n).overflow;
+ if (o === 'visible') continue;
+ const c = n.getBoundingClientRect();
+ if (x >= c.left && x < c.right && y >= c.top && y < c.bottom) continue;
+ if (retried || !(/auto|scroll/.test(o) || n.scrollHeight > n.clientHeight || n.scrollWidth > n.clientWidth)) return null;
+ try { t.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { /* keep */ }
+ return centreHit(t, true);
+ }
+ return false;
+ };
+ // Opacity does not stop hit-testing and multiplies down the tree, so a faded-out panel, or stacked
+ // translucent wrappers, hide an input that elementFromPoint still returns. The input's own is excluded.
+ const fadedAncestry = () => {
+ let o = 1;
+ for (let n = el.assignedSlot || el.parentNode || el.host, hops = 0; n && hops < 256; hops++, n = n.assignedSlot || n.parentNode || n.host || null) {
+ if (n.nodeType === 1) o *= parseFloat(getComputedStyle(n).opacity || '1');
+ }
+ return o < 0.05;
+ };
+ if (!_nativeProxy(el)) {
+ // Only the input's own transparency is waived: it still needs a box, no visibility:hidden, visible
+ // ancestors, and to be on top at its own centre, so it is clicked directly and read back as a toggle.
+ const at = r.width > 0 && r.height > 0 && cs.visibility !== 'hidden' && !fadedAncestry() ? centreHit(el) : null;
+ if (at) return { exists: true, skinned: false, labelClick: null, ...toggleFields(_toggleOwner(el)) };
+ if (at === false) return { exists: true, skinned: false, labelClick: null, radio, covered: true };
+ return { exists: true, skinned: false, labelClick: null, radio, unproxied: true };
+ }
const disabled = !!el.disabled;
const none = { exists: true, skinned: true, labelClick: null, radio, disabled };
+ // `none` still gets a script click, so a proxy inside a faded panel is refused outright.
+ if (fadedAncestry()) return { ...none, unproxied: true };
// Only a real <label> activates its control on click, and only the association THIS realm derives
// counts: an `el.labels` the page shadows names a decoy, not a proxy. A realm the page can patch
// (the main-world fallback) never offers one.
@@ -5921,31 +5977,9 @@ _SKINNED_CHECKBOX_PROBE_JS = (
return b.width > 0 && b.height > 0 && !wrapsOther(l);
}) || null;
if (!label || nativeControlOf(label) !== el) return none;
- // The click lands on coordinates this realm measured, never through a marker the page could move
- // onto something else, so the point has to be the label's own: a cover there is a cover.
- let b = label.getBoundingClientRect();
- if (b.bottom < 0 || b.right < 0 || b.top > innerHeight || b.left > innerWidth) {
- try { label.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' }); } catch (e) { /* keep */ }
- b = label.getBoundingClientRect();
- }
- const x = b.left + b.width / 2, y = b.top + b.height / 2;
- let hit = null;
- try { hit = document.elementFromPoint(x, y); } catch (e) { hit = null; }
- for (let hops = 0; hit && hops < 32; hops++) {
- let root = null;
- try { root = hit.shadowRoot; } catch (e) { break; }
- if (!root || root.nodeType !== 11) break;
- let inner = null;
- try { inner = root.elementFromPoint(x, y); } catch (e) { break; }
- if (!inner || inner === hit) break;
- hit = inner;
- }
- let onLabel = false;
- for (let n = hit, hops = 0; n && hops < 256; hops++, n = n.assignedSlot || n.parentNode || n.host || null) {
- if (n === label || n === el) { onLabel = true; break; }
- }
- if (!onLabel) return { ...none, labelCovered: true };
- return { exists: true, skinned: true, labelClick: { x, y }, radio, disabled };
+ const at = centreHit(label);
+ if (!at) return { ...none, labelCovered: true };
+ return { exists: true, skinned: true, labelClick: at, radio, disabled };
} catch (e) { return { exists: false, skinned: false, labelClick: null }; }
}"""
)
@@ -12958,7 +12992,7 @@ def build_browser_tools(
if isinstance(skin_probe, dict) and skin_probe.get("unproxied"):
return await _unreachable_error(selector)
skinned = bool(isinstance(skin_probe, dict) and skin_probe.get("skinned"))
- if skinned and skin_probe.get("labelCovered"):
+ if isinstance(skin_probe, dict) and (skin_probe.get("covered") or (skinned and skin_probe.get("labelCovered"))):
return _covered_error(selector, None, verb="clicked")
label_click = skin_probe.get("labelClick") if isinstance(skin_probe, dict) else None
if not isinstance(label_click, dict):
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 98b9a8552..45664ed4d 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -14761,6 +14761,88 @@ async def test_click_refuses_a_skinned_checkbox_with_no_proxy_at_all() -> None:
assert await page.eval_on_selector("#nolabel", "el => el.checked") is False
+# A transparent native checkbox stretched over its own styled box is what a person clicks, in view, below
+# the fold, with only its top edge above the fold, or further down a scrollable list. One inside a collapsed
+# section, a faded-out panel (even behind its <label>), or stacked translucent wrappers, or under an unrelated
+# cover, is not reachable, and the two are told apart.
+_OPACITY0_OVERLAY_CHECKBOX_HTML = """
+<!doctype html><html><body style="margin:0">
+ <div style="position:relative;width:24px;height:24px;margin:20px">
+ <span style="position:absolute;inset:0;border:2px solid #333;pointer-events:none"></span>
+ <input id="overlay" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0;z-index:1">
+ </div>
+ <div style="height:0;overflow:hidden">
+ <div style="position:relative;width:24px;height:24px">
+ <input id="collapsed" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0">
+ </div>
+ </div>
+ <div style="position:relative;width:24px;height:24px;margin:20px">
+ <input id="covered" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0">
+ <div style="position:absolute;inset:0;z-index:2;background:#fff">Show more</div>
+ </div>
+ <div style="position:relative;width:24px;height:24px;margin:20px;opacity:0">
+ <input id="faded" type="checkbox" aria-label="Faded panel option" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0">
+ </div>
+ <div style="opacity:0;margin:20px">
+ <label style="display:inline-block;width:120px;height:24px"><input id="labelfaded" type="checkbox" style="opacity:0">Faded label</label>
+ </div>
+ <div style="opacity:0.2;margin:20px">
+ <div style="position:relative;width:24px;height:24px;opacity:0.2">
+ <input id="stacked" type="checkbox" aria-label="Stacked fade option" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0">
+ </div>
+ </div>
+ <div style="height:100px;overflow:auto;margin:20px">
+ <div style="height:600px"></div>
+ <div style="position:relative;width:24px;height:24px">
+ <span style="position:absolute;inset:0;border:2px solid #333;pointer-events:none"></span>
+ <input id="inlist" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0;z-index:1">
+ </div>
+ </div>
+ <div style="position:absolute;left:20px;top:calc(100vh - 8px);width:24px;height:24px">
+ <span style="position:absolute;inset:0;border:2px solid #333;pointer-events:none"></span>
+ <input id="straddle" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0;z-index:1">
+ </div>
+ <div style="height:3000px"></div>
+ <div style="position:relative;width:24px;height:24px;margin:20px">
+ <span style="position:absolute;inset:0;border:2px solid #333;pointer-events:none"></span>
+ <input id="below" type="checkbox" style="position:absolute;inset:0;width:100%;height:100%;margin:0;opacity:0;z-index:1">
+ </div>
+</body></html>
+"""
+
+
+@_skip_no_browser
[email protected]("selector", ["#overlay", "#below", "#straddle", "#inlist"])
+async def test_click_checks_a_transparent_checkbox_laid_over_its_own_styled_box(selector: str) -> None:
+ async with _content_page(_OPACITY0_OVERLAY_CHECKBOX_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": selector})
+ assert r.status == "ok", r.content
+ assert await page.eval_on_selector(selector, "el => el.checked") is True
+
+
+@_skip_no_browser
+ ("selector", "diagnosis"),
+ [
+ ("#collapsed", "not rendered"),
+ ("#faded", "not rendered"),
+ ("#labelfaded", "not rendered"),
+ ("#stacked", "not rendered"),
+ ("#covered", "something else is on top of it"),
+ ],
+)
+async def test_click_still_refuses_a_transparent_checkbox_nothing_reaches(selector: str, diagnosis: str) -> None:
+ async with _content_page(_OPACITY0_OVERLAY_CHECKBOX_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": selector})
+ assert r.status == "error"
+ assert diagnosis in r.content, r.content
+ assert await page.eval_on_selector(selector, "el => el.checked") is False
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_a_pre_seeded_marker_holding_a_line_separator_cannot_forge_a_line() -> None: