Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: handle clipped scrollers and shadow host occlusion
Barack Sokulluab5ef008596d
diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.jsindex 010ab2ba5..5fa359cdf 100644--- a/src/chrome/src/content/accessibility-tree.js+++ b/src/chrome/src/content/accessibility-tree.js@@ -586,6 +586,17 @@: null;}+ function isComposedAncestor(ancestor, node) {+ for (let current = node; current; current = composedParent(current)) {+ if (current === ancestor) return true;+ }+ return false;+ }++ function isComposedRelated(a, b) {+ return isComposedAncestor(a, b) || isComposedAncestor(b, a);+ }+function deepestOpenShadowHit(x, y) {let hit = document.elementFromPoint(x, y);const seen = new Set();@@ -611,7 +622,7 @@const cy = r.top + r.height / 2;if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {const hit = deepestOpenShadowHit(cx, cy);- if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;+ if (hit && isComposedRelated(el, hit)) return false;}const sampleOffsets = [@@ -623,7 +634,7 @@for (const [sx, sy] of sampleOffsets) {if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;const sampleHit = deepestOpenShadowHit(sx, sy);- if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {+ if (sampleHit && isComposedRelated(el, sampleHit)) {return false;}}@@ -635,7 +646,7 @@const allAboveNone = elements.slice(0, targetIdx).every(item => {try {const pe = window.getComputedStyle(item).pointerEvents;- return pe === 'none' || el.contains(item) || item.contains(el);+ return pe === 'none' || isComposedRelated(el, item);} catch {return false;}diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 1c08826ae..50667e479 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -358,16 +358,9 @@try {const view = el.ownerDocument?.defaultView || window;const insets = _getViewportDockedInsets(view);- const rBefore = el.getBoundingClientRect();- const inClear = (- rBefore.width >= 1- && rBefore.height >= 1- && rBefore.left >= 0- && rBefore.right <= view.innerWidth- && rBefore.top >= insets.top- && rBefore.bottom <= (view.innerHeight - insets.bottom)- );- if (!inClear) {+ // A viewport-only rect check misses elements clipped by a scrollable+ // ancestor even when their bounding box is inside the viewport.+ if (!_isFullyVisibleForInteraction(el, insets)) {el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });const rAfter = el.getBoundingClientRect();if (rAfter.top < insets.top) {@@ -381,12 +374,12 @@}}- function _isFullyVisibleForInteraction(el) {+ function _isFullyVisibleForInteraction(el, dockedInsets = null) {try {if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();- const insets = _getViewportDockedInsets(view);+ const insets = dockedInsets || _getViewportDockedInsets(view);if (rect.width < 1|| rect.height < 1diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.jsindex 010ab2ba5..5fa359cdf 100644--- a/src/firefox/src/content/accessibility-tree.js+++ b/src/firefox/src/content/accessibility-tree.js@@ -586,6 +586,17 @@: null;}+ function isComposedAncestor(ancestor, node) {+ for (let current = node; current; current = composedParent(current)) {+ if (current === ancestor) return true;+ }+ return false;+ }++ function isComposedRelated(a, b) {+ return isComposedAncestor(a, b) || isComposedAncestor(b, a);+ }+function deepestOpenShadowHit(x, y) {let hit = document.elementFromPoint(x, y);const seen = new Set();@@ -611,7 +622,7 @@const cy = r.top + r.height / 2;if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {const hit = deepestOpenShadowHit(cx, cy);- if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;+ if (hit && isComposedRelated(el, hit)) return false;}const sampleOffsets = [@@ -623,7 +634,7 @@for (const [sx, sy] of sampleOffsets) {if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;const sampleHit = deepestOpenShadowHit(sx, sy);- if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {+ if (sampleHit && isComposedRelated(el, sampleHit)) {return false;}}@@ -635,7 +646,7 @@const allAboveNone = elements.slice(0, targetIdx).every(item => {try {const pe = window.getComputedStyle(item).pointerEvents;- return pe === 'none' || el.contains(item) || item.contains(el);+ return pe === 'none' || isComposedRelated(el, item);} catch {return false;}diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 084bed2ec..0291ba6ca 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -567,16 +567,9 @@try {const view = el.ownerDocument?.defaultView || window;const insets = _getViewportDockedInsets(view);- const rBefore = el.getBoundingClientRect();- const inClear = (- rBefore.width >= 1- && rBefore.height >= 1- && rBefore.left >= 0- && rBefore.right <= view.innerWidth- && rBefore.top >= insets.top- && rBefore.bottom <= (view.innerHeight - insets.bottom)- );- if (!inClear) {+ // A viewport-only rect check misses elements clipped by a scrollable+ // ancestor even when their bounding box is inside the viewport.+ if (!_isFullyVisibleForInteraction(el, insets)) {el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });const rAfter = el.getBoundingClientRect();if (rAfter.top < insets.top) {@@ -590,12 +583,12 @@}}- function _isFullyVisibleForInteraction(el) {+ function _isFullyVisibleForInteraction(el, dockedInsets = null) {try {if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();- const insets = _getViewportDockedInsets(view);+ const insets = dockedInsets || _getViewportDockedInsets(view);if (rect.width < 1|| rect.height < 1diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjsindex cca815095..34fcdb021 100644--- a/test/fixtures/run.mjs+++ b/test/fixtures/run.mjs@@ -2181,6 +2181,74 @@ test('Chrome Agent: modal auto-select ignores background/hidden clickables and k});// βββ occlusion ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+async function assertInnerScrollerClearance(page, browserKind) {+ await setupContentHtml(page, `<!doctype html>+ <style>+ body { margin: 0; }+ #scroller { margin: 20px; width: 260px; height: 100px; overflow: auto; }+ #target { display: block; margin-top: 200px; width: 120px; height: 32px; }+ </style>+ <div id="scroller"><button id="target" onclick="window.__targetClicked = true">Target</button></div>+ `, browserKind);+ const before = await page.evaluate(() => {+ const scroller = document.getElementById('scroller');+ const target = document.getElementById('target');+ return {+ scrollTop: scroller.scrollTop,+ targetTop: target.getBoundingClientRect().top,+ scrollerBottom: scroller.getBoundingClientRect().bottom,+ viewportHeight: window.innerHeight,+ };+ });+ if (before.scrollTop !== 0 || before.targetTop <= before.scrollerBottom || before.targetTop >= before.viewportHeight) {+ throw new Error(`fixture must be clipped by its scroller but inside the viewport: ${JSON.stringify(before)}`);+ }+ const result = await call(page, 'click', { selector: '#target' });+ const after = await page.evaluate(() => ({+ scrollTop: document.getElementById('scroller').scrollTop,+ clicked: window.__targetClicked === true,+ }));+ if (!result?.success || !after.clicked || after.scrollTop <= 0) {+ throw new Error(`${browserKind} click did not expose the clipped target: ${JSON.stringify({ result, after })}`);+ }+}++test('Chrome: click scrolls a target clipped by an inner scroller', page =>+ assertInnerScrollerClearance(page, 'chrome'));+firefoxTest('Firefox: click scrolls a target clipped by an inner scroller', page =>+ assertInnerScrollerClearance(page, 'firefox'));++async function assertShadowHostOcclusion(page, sourcePath) {+ await setupAccessibilityTreeHtml(page, `<!doctype html>+ <style>+ body { margin: 0; }+ #host { position: absolute; left: 20px; top: 20px; width: 200px; height: 50px; }+ </style>+ <div id="host" role="button" tabindex="0" aria-label="Shadow control"></div>+ <script>+ document.getElementById('host').attachShadow({ mode: 'open' }).innerHTML =+ '<span style="display:block;width:100%;height:100%;background:green">Visible content</span>';+ </script>+ `, sourcePath);+ const result = await page.evaluate(() => {+ const line = () => String(window.__generateAccessibilityTree('all', 5, 20000).pageContent || '')+ .split('\n').find(item => item.includes('"Shadow control"')) || '';+ const visible = line();+ const cover = document.createElement('div');+ cover.style.cssText = 'position:absolute;left:20px;top:20px;width:200px;height:50px;z-index:2;background:red';+ document.body.append(cover);+ return { visible, covered: line() };+ });+ if (!result.visible || result.visible.includes('occluded=true') || !result.covered.includes('occluded=true')) {+ throw new Error(`shadow host occlusion state is wrong: ${JSON.stringify(result)}`);+ }+}++test('Chrome: visible shadow host is not marked occluded', page =>+ assertShadowHostOcclusion(page, accessibilityTreeJsPath));+firefoxTest('Firefox: visible shadow host is not marked occluded', page =>+ assertShadowHostOcclusion(page, firefoxAccessibilityTreeJsPath));+test('occlusion: click({text:"Submit"}) refuses when covered', async (page) => {await setup(page, 'occlusion.html');const resp = await call(page, 'click', { text: 'Submit' });