Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: handle clipped scrollers and shadow host occlusion

Barack Sokulluab5ef008596d
diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.js
index 010ab2ba5..5fa359cdf 100644
--- a/src/chrome/src/content/accessibility-tree.js
+++ b/src/chrome/src/content/accessibility-tree.js
@@ -586,6 +586,17 @@
: null;
}
+ function isComposedAncestor(ancestor, node) {
+ for (let current = node; current; current = composedParent(current)) {
+ if (current === ancestor) return true;
+ }
+ return false;
+ }
+
+ function isComposedRelated(a, b) {
+ return isComposedAncestor(a, b) || isComposedAncestor(b, a);
+ }
+
function deepestOpenShadowHit(x, y) {
let hit = document.elementFromPoint(x, y);
const seen = new Set();
@@ -611,7 +622,7 @@
const cy = r.top + r.height / 2;
if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
const hit = deepestOpenShadowHit(cx, cy);
- if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;
+ if (hit && isComposedRelated(el, hit)) return false;
}
const sampleOffsets = [
@@ -623,7 +634,7 @@
for (const [sx, sy] of sampleOffsets) {
if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
const sampleHit = deepestOpenShadowHit(sx, sy);
- if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {
+ if (sampleHit && isComposedRelated(el, sampleHit)) {
return false;
}
}
@@ -635,7 +646,7 @@
const allAboveNone = elements.slice(0, targetIdx).every(item => {
try {
const pe = window.getComputedStyle(item).pointerEvents;
- return pe === 'none' || el.contains(item) || item.contains(el);
+ return pe === 'none' || isComposedRelated(el, item);
} catch {
return false;
}
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 1c08826ae..50667e479 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -358,16 +358,9 @@
try {
const view = el.ownerDocument?.defaultView || window;
const insets = _getViewportDockedInsets(view);
- const rBefore = el.getBoundingClientRect();
- const inClear = (
- rBefore.width >= 1
- && rBefore.height >= 1
- && rBefore.left >= 0
- && rBefore.right <= view.innerWidth
- && rBefore.top >= insets.top
- && rBefore.bottom <= (view.innerHeight - insets.bottom)
- );
- if (!inClear) {
+ // A viewport-only rect check misses elements clipped by a scrollable
+ // ancestor even when their bounding box is inside the viewport.
+ if (!_isFullyVisibleForInteraction(el, insets)) {
el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });
const rAfter = el.getBoundingClientRect();
if (rAfter.top < insets.top) {
@@ -381,12 +374,12 @@
}
}
- function _isFullyVisibleForInteraction(el) {
+ function _isFullyVisibleForInteraction(el, dockedInsets = null) {
try {
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
- const insets = _getViewportDockedInsets(view);
+ const insets = dockedInsets || _getViewportDockedInsets(view);
if (
rect.width < 1
|| rect.height < 1
diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.js
index 010ab2ba5..5fa359cdf 100644
--- a/src/firefox/src/content/accessibility-tree.js
+++ b/src/firefox/src/content/accessibility-tree.js
@@ -586,6 +586,17 @@
: null;
}
+ function isComposedAncestor(ancestor, node) {
+ for (let current = node; current; current = composedParent(current)) {
+ if (current === ancestor) return true;
+ }
+ return false;
+ }
+
+ function isComposedRelated(a, b) {
+ return isComposedAncestor(a, b) || isComposedAncestor(b, a);
+ }
+
function deepestOpenShadowHit(x, y) {
let hit = document.elementFromPoint(x, y);
const seen = new Set();
@@ -611,7 +622,7 @@
const cy = r.top + r.height / 2;
if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
const hit = deepestOpenShadowHit(cx, cy);
- if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;
+ if (hit && isComposedRelated(el, hit)) return false;
}
const sampleOffsets = [
@@ -623,7 +634,7 @@
for (const [sx, sy] of sampleOffsets) {
if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
const sampleHit = deepestOpenShadowHit(sx, sy);
- if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {
+ if (sampleHit && isComposedRelated(el, sampleHit)) {
return false;
}
}
@@ -635,7 +646,7 @@
const allAboveNone = elements.slice(0, targetIdx).every(item => {
try {
const pe = window.getComputedStyle(item).pointerEvents;
- return pe === 'none' || el.contains(item) || item.contains(el);
+ return pe === 'none' || isComposedRelated(el, item);
} catch {
return false;
}
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 084bed2ec..0291ba6ca 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -567,16 +567,9 @@
try {
const view = el.ownerDocument?.defaultView || window;
const insets = _getViewportDockedInsets(view);
- const rBefore = el.getBoundingClientRect();
- const inClear = (
- rBefore.width >= 1
- && rBefore.height >= 1
- && rBefore.left >= 0
- && rBefore.right <= view.innerWidth
- && rBefore.top >= insets.top
- && rBefore.bottom <= (view.innerHeight - insets.bottom)
- );
- if (!inClear) {
+ // A viewport-only rect check misses elements clipped by a scrollable
+ // ancestor even when their bounding box is inside the viewport.
+ if (!_isFullyVisibleForInteraction(el, insets)) {
el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });
const rAfter = el.getBoundingClientRect();
if (rAfter.top < insets.top) {
@@ -590,12 +583,12 @@
}
}
- function _isFullyVisibleForInteraction(el) {
+ function _isFullyVisibleForInteraction(el, dockedInsets = null) {
try {
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
- const insets = _getViewportDockedInsets(view);
+ const insets = dockedInsets || _getViewportDockedInsets(view);
if (
rect.width < 1
|| rect.height < 1
diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs
index cca815095..34fcdb021 100644
--- a/test/fixtures/run.mjs
+++ b/test/fixtures/run.mjs
@@ -2181,6 +2181,74 @@ test('Chrome Agent: modal auto-select ignores background/hidden clickables and k
});
// ─── occlusion ────────────────────────────────────────────────────────────
+async function assertInnerScrollerClearance(page, browserKind) {
+ await setupContentHtml(page, `<!doctype html>
+ <style>
+ body { margin: 0; }
+ #scroller { margin: 20px; width: 260px; height: 100px; overflow: auto; }
+ #target { display: block; margin-top: 200px; width: 120px; height: 32px; }
+ </style>
+ <div id="scroller"><button id="target" onclick="window.__targetClicked = true">Target</button></div>
+ `, browserKind);
+ const before = await page.evaluate(() => {
+ const scroller = document.getElementById('scroller');
+ const target = document.getElementById('target');
+ return {
+ scrollTop: scroller.scrollTop,
+ targetTop: target.getBoundingClientRect().top,
+ scrollerBottom: scroller.getBoundingClientRect().bottom,
+ viewportHeight: window.innerHeight,
+ };
+ });
+ if (before.scrollTop !== 0 || before.targetTop <= before.scrollerBottom || before.targetTop >= before.viewportHeight) {
+ throw new Error(`fixture must be clipped by its scroller but inside the viewport: ${JSON.stringify(before)}`);
+ }
+ const result = await call(page, 'click', { selector: '#target' });
+ const after = await page.evaluate(() => ({
+ scrollTop: document.getElementById('scroller').scrollTop,
+ clicked: window.__targetClicked === true,
+ }));
+ if (!result?.success || !after.clicked || after.scrollTop <= 0) {
+ throw new Error(`${browserKind} click did not expose the clipped target: ${JSON.stringify({ result, after })}`);
+ }
+}
+
+test('Chrome: click scrolls a target clipped by an inner scroller', page =>
+ assertInnerScrollerClearance(page, 'chrome'));
+firefoxTest('Firefox: click scrolls a target clipped by an inner scroller', page =>
+ assertInnerScrollerClearance(page, 'firefox'));
+
+async function assertShadowHostOcclusion(page, sourcePath) {
+ await setupAccessibilityTreeHtml(page, `<!doctype html>
+ <style>
+ body { margin: 0; }
+ #host { position: absolute; left: 20px; top: 20px; width: 200px; height: 50px; }
+ </style>
+ <div id="host" role="button" tabindex="0" aria-label="Shadow control"></div>
+ <script>
+ document.getElementById('host').attachShadow({ mode: 'open' }).innerHTML =
+ '<span style="display:block;width:100%;height:100%;background:green">Visible content</span>';
+ </script>
+ `, sourcePath);
+ const result = await page.evaluate(() => {
+ const line = () => String(window.__generateAccessibilityTree('all', 5, 20000).pageContent || '')
+ .split('\n').find(item => item.includes('"Shadow control"')) || '';
+ const visible = line();
+ const cover = document.createElement('div');
+ cover.style.cssText = 'position:absolute;left:20px;top:20px;width:200px;height:50px;z-index:2;background:red';
+ document.body.append(cover);
+ return { visible, covered: line() };
+ });
+ if (!result.visible || result.visible.includes('occluded=true') || !result.covered.includes('occluded=true')) {
+ throw new Error(`shadow host occlusion state is wrong: ${JSON.stringify(result)}`);
+ }
+}
+
+test('Chrome: visible shadow host is not marked occluded', page =>
+ assertShadowHostOcclusion(page, accessibilityTreeJsPath));
+firefoxTest('Firefox: visible shadow host is not marked occluded', page =>
+ assertShadowHostOcclusion(page, firefoxAccessibilityTreeJsPath));
+
test('occlusion: click({text:"Submit"}) refuses when covered', async (page) => {
await setup(page, 'occlusion.html');
const resp = await call(page, 'click', { text: 'Submit' });