Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: mark ancestor pseudo-element covers as occluded

Barack Sokulluf7450c6a651d
diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.js
index d3c8eb580..38070d700 100644
--- a/src/chrome/src/content/accessibility-tree.js
+++ b/src/chrome/src/content/accessibility-tree.js
@@ -593,10 +593,6 @@
return false;
}
- function isComposedRelated(a, b) {
- return isComposedAncestor(a, b) || isComposedAncestor(b, a);
- }
-
function deepestOpenShadowHit(x, y) {
let hit = document.elementFromPoint(x, y);
const seen = new Set();
@@ -622,7 +618,7 @@
const cy = r.top + r.height / 2;
if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
const hit = deepestOpenShadowHit(cx, cy);
- if (hit && isComposedRelated(el, hit)) return false;
+ if (hit && isComposedAncestor(el, hit)) return false;
}
const sampleOffsets = [
@@ -634,7 +630,7 @@
for (const [sx, sy] of sampleOffsets) {
if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
const sampleHit = deepestOpenShadowHit(sx, sy);
- if (sampleHit && isComposedRelated(el, sampleHit)) {
+ if (sampleHit && isComposedAncestor(el, sampleHit)) {
return false;
}
}
@@ -646,7 +642,7 @@
const allAboveNone = elements.slice(0, targetIdx).every(item => {
try {
const pe = window.getComputedStyle(item).pointerEvents;
- return pe === 'none' || isComposedRelated(el, item);
+ return pe === 'none' || isComposedAncestor(el, item);
} catch {
return false;
}
diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.js
index d3c8eb580..38070d700 100644
--- a/src/firefox/src/content/accessibility-tree.js
+++ b/src/firefox/src/content/accessibility-tree.js
@@ -593,10 +593,6 @@
return false;
}
- function isComposedRelated(a, b) {
- return isComposedAncestor(a, b) || isComposedAncestor(b, a);
- }
-
function deepestOpenShadowHit(x, y) {
let hit = document.elementFromPoint(x, y);
const seen = new Set();
@@ -622,7 +618,7 @@
const cy = r.top + r.height / 2;
if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
const hit = deepestOpenShadowHit(cx, cy);
- if (hit && isComposedRelated(el, hit)) return false;
+ if (hit && isComposedAncestor(el, hit)) return false;
}
const sampleOffsets = [
@@ -634,7 +630,7 @@
for (const [sx, sy] of sampleOffsets) {
if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
const sampleHit = deepestOpenShadowHit(sx, sy);
- if (sampleHit && isComposedRelated(el, sampleHit)) {
+ if (sampleHit && isComposedAncestor(el, sampleHit)) {
return false;
}
}
@@ -646,7 +642,7 @@
const allAboveNone = elements.slice(0, targetIdx).every(item => {
try {
const pe = window.getComputedStyle(item).pointerEvents;
- return pe === 'none' || isComposedRelated(el, item);
+ return pe === 'none' || isComposedAncestor(el, item);
} catch {
return false;
}
diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs
index d225e3763..6e0e23496 100644
--- a/test/fixtures/run.mjs
+++ b/test/fixtures/run.mjs
@@ -2655,6 +2655,35 @@ test('Chrome: visible shadow host is not marked occluded', page =>
firefoxTest('Firefox: visible shadow host is not marked occluded', page =>
assertShadowHostOcclusion(page, firefoxAccessibilityTreeJsPath));
+async function assertAncestorPseudoElementOcclusion(page, sourcePath, label) {
+ await setupAccessibilityTreeHtml(page, `<!doctype html>
+ <style>
+ #cover-parent { position: relative; margin: 100px; width: 220px; height: 100px; }
+ #cover-parent::after { content: ''; position: absolute; inset: 0; z-index: 2; background: rgba(255,255,255,.1); pointer-events: auto; }
+ #target { position: absolute; top: 20px; left: 20px; width: 120px; height: 40px; }
+ </style>
+ <div id="cover-parent"><button id="target">Pseudo covered action</button></div>
+ `, sourcePath);
+ const state = await page.evaluate(() => {
+ const target = document.getElementById('target');
+ const rect = target.getBoundingClientRect();
+ const tree = window.__generateAccessibilityTree('all', 5, 20000);
+ return {
+ hitIsParent: document.elementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2)
+ === document.getElementById('cover-parent'),
+ line: String(tree?.pageContent || '').split('\n').find(item => item.includes('"Pseudo covered action"')) || '',
+ };
+ });
+ if (!state.hitIsParent || !state.line.includes('occluded=true')) {
+ throw new Error(`${label}: ancestor pseudo-element was not marked as covering the target: ${JSON.stringify(state)}`);
+ }
+}
+
+test('Chrome: ancestor pseudo-element marks its child control occluded', page =>
+ assertAncestorPseudoElementOcclusion(page, accessibilityTreeJsPath, 'chrome'));
+firefoxTest('Firefox: ancestor pseudo-element marks its child control occluded', page =>
+ assertAncestorPseudoElementOcclusion(page, firefoxAccessibilityTreeJsPath, 'firefox'));
+
test('occlusion: click({text:"Submit"}) refuses when covered', async (page) => {
await setup(page, 'occlusion.html');
const resp = await call(page, 'click', { text: 'Submit' });