Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: handle overlapping content elements and docked headers (#3085)

Emre Sokullubda9ae760b5c
diff --git a/src/chrome/src/agent/loop-detector.js b/src/chrome/src/agent/loop-detector.js
index 44aa69cab..bd93586ca 100644
--- a/src/chrome/src/agent/loop-detector.js
+++ b/src/chrome/src/agent/loop-detector.js
@@ -621,21 +621,28 @@ export class LoopDetector {
}
const failures = this.failedActionLoops.get(tabId) || new Map();
if (this._isToolResultErroredForLoop(toolName, toolArgs, toolResult)) {
+ const isOcclusionFailure = toolResult?.occluded === true
+ || /occluded|covered by another element|topmost/i.test(String(toolResult?.error || ''));
+ const maxAttempts = isOcclusionFailure ? 4 : 3;
const attempts = (failures.get(failureScope) || 0) + 1;
failures.set(failureScope, attempts);
if (failures.size > 32) failures.delete(failures.keys().next().value);
this.failedActionLoops.set(tabId, failures);
- if (attempts >= 3) {
+ if (attempts >= maxAttempts) {
this._clearLoopState(tabId);
return {
kind: 'stop',
- message: `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,
+ message: isOcclusionFailure
+ ? `Stopped: ${toolName} failed ${maxAttempts} times because the target is occluded by an overlapping layer or docked header. Close the overlay, scroll past the obstruction, or re-read the page before retrying.`
+ : `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,
};
}
- if (attempts === 2) {
+ if (attempts >= 2) {
return {
kind: 'nudge',
- warning: `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,
+ warning: isOcclusionFailure
+ ? `[OVERLAY OCCLUSION: ${toolName} failed because the target element is occluded or covered by an overlapping element (such as a modal backdrop, fixed header, or sticky toolbar). Try closing the overlay (e.g. clicking a close/dismiss button or pressing Escape), scrolling the element into clear view, or re-reading the accessibility tree.]`
+ : `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,
};
}
} else if (toolResult?.success === true && toolResult?.verified !== false) {
diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.js
index c85f30c57..010ab2ba5 100644
--- a/src/chrome/src/content/accessibility-tree.js
+++ b/src/chrome/src/content/accessibility-tree.js
@@ -598,6 +598,58 @@
return hit;
}
+ function isNodeOccluded(el) {
+ if (!el || !el.isConnected || el.nodeType !== Node.ELEMENT_NODE) return false;
+ try {
+ const r = el.getBoundingClientRect();
+ if (r.width < 1 || r.height < 1) return false;
+ const vw = window.innerWidth || 800;
+ const vh = window.innerHeight || 600;
+ if (r.top >= vh || r.bottom <= 0 || r.left >= vw || r.right <= 0) return false;
+
+ const cx = r.left + r.width / 2;
+ const cy = r.top + r.height / 2;
+ if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
+ const hit = deepestOpenShadowHit(cx, cy);
+ if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;
+ }
+
+ const sampleOffsets = [
+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],
+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],
+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ ];
+ for (const [sx, sy] of sampleOffsets) {
+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
+ const sampleHit = deepestOpenShadowHit(sx, sy);
+ if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {
+ return false;
+ }
+ }
+
+ if (document.elementsFromPoint && cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
+ const elements = document.elementsFromPoint(cx, cy);
+ const targetIdx = elements.indexOf(el);
+ if (targetIdx > 0) {
+ const allAboveNone = elements.slice(0, targetIdx).every(item => {
+ try {
+ const pe = window.getComputedStyle(item).pointerEvents;
+ return pe === 'none' || el.contains(item) || item.contains(el);
+ } catch {
+ return false;
+ }
+ });
+ if (allAboveNone) return false;
+ }
+ }
+
+ return true;
+ } catch {
+ return false;
+ }
+ }
+
function visualTargetEligibility(el) {
const tag = el.tagName?.toLowerCase() || '';
if (tag === 'button') return 'semantic-button';
@@ -879,6 +931,11 @@
}
} catch {}
}
+ try {
+ if (isInteractive(el) && isNodeOccluded(el)) {
+ line += ' occluded=true';
+ }
+ } catch {}
if (inputType === 'checkbox' || inputType === 'radio') {
line += ` checked=${el.checked ? 'true' : 'false'}`;
} else if (['checkbox', 'radio', 'switch'].includes(attrRole) || el.hasAttribute('aria-checked')) {
@@ -1565,6 +1622,13 @@
'[aria-modal="true"]',
'[role=combobox][aria-expanded="true"]',
'dialog[open]',
+ '[data-overlay]',
+ '.modal.show',
+ '.modal-overlay',
+ '[class*="modal"][class*="open"]',
+ '[class*="DialogContent"]',
+ '[class*="ModalContent"]',
+ '[data-state="open"][role="dialog"]',
];
const overlayEls = [];
const seen = new WeakSet();
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 82d28f8ff..1c08826ae 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -311,18 +311,89 @@
: null;
}
+ function _getViewportDockedInsets(view = window) {
+ let top = 0;
+ let bottom = 0;
+ try {
+ const doc = view.document;
+ if (!doc) return { top: 0, bottom: 0 };
+ const htmlStyle = view.getComputedStyle(doc.documentElement);
+ const bodyStyle = doc.body ? view.getComputedStyle(doc.body) : null;
+ const parsePadding = val => {
+ const n = parseFloat(val);
+ return Number.isFinite(n) && n > 0 ? n : 0;
+ };
+ top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
+ bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
+ if (bodyStyle) {
+ top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
+ bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
+ }
+ const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');
+ const vw = view.innerWidth || 800;
+ const vh = view.innerHeight || 600;
+ for (const c of candidates) {
+ if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;
+ const cs = view.getComputedStyle(c);
+ if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
+ if (cs.display === 'none' || cs.visibility === 'hidden') continue;
+ const r = c.getBoundingClientRect();
+ if (r.width < vw * 0.4) continue;
+ if (r.top <= 10 && r.bottom > 0 && r.bottom < vh * 0.4) {
+ if (r.bottom > top) top = r.bottom;
+ } else if (r.bottom >= vh - 10 && r.top < vh && r.top > vh * 0.6) {
+ const h = vh - r.top;
+ if (h > bottom) bottom = h;
+ }
+ }
+ } catch {}
+ return {
+ top: Math.min(top, (view.innerHeight || 600) * 0.4),
+ bottom: Math.min(bottom, (view.innerHeight || 600) * 0.4),
+ };
+ }
+
+ function _scrollElementIntoClearView(el) {
+ if (!el?.isConnected) return;
+ try {
+ const view = el.ownerDocument?.defaultView || window;
+ const insets = _getViewportDockedInsets(view);
+ const rBefore = el.getBoundingClientRect();
+ const inClear = (
+ rBefore.width >= 1
+ && rBefore.height >= 1
+ && rBefore.left >= 0
+ && rBefore.right <= view.innerWidth
+ && rBefore.top >= insets.top
+ && rBefore.bottom <= (view.innerHeight - insets.bottom)
+ );
+ if (!inClear) {
+ el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });
+ const rAfter = el.getBoundingClientRect();
+ if (rAfter.top < insets.top) {
+ view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });
+ } else if (rAfter.bottom > view.innerHeight - insets.bottom) {
+ view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });
+ }
+ }
+ } catch {
+ try { el.scrollIntoView({ block: 'center', inline: 'nearest' }); } catch {}
+ }
+ }
+
function _isFullyVisibleForInteraction(el) {
try {
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
+ const insets = _getViewportDockedInsets(view);
if (
rect.width < 1
|| rect.height < 1
|| rect.left < 0
- || rect.top < 0
+ || rect.top < insets.top
|| rect.right > view.innerWidth
- || rect.bottom > view.innerHeight
+ || rect.bottom > (view.innerHeight - insets.bottom)
) return false;
for (let node = _composedParent(el); node && node !== el.ownerDocument; node = _composedParent(node)) {
if (node.nodeType !== Node.ELEMENT_NODE) continue;
@@ -1893,7 +1964,7 @@
// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)
if (el.tagName !== 'SELECT') {
if (actionDeadlineExpired()) return deadlineFailure();
- el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+ _scrollElementIntoClearView(el);
}
// Occlusion hit-test: for text/selector/index clicks, verify that the
@@ -1906,11 +1977,59 @@
// elements with real bounding rects.
if (el.tagName !== 'SELECT' && params.x == null && params.y == null) {
try {
- const r = el.getBoundingClientRect();
+ let r = el.getBoundingClientRect();
if (r.width >= 1 && r.height >= 1 && r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth) {
- const cx = Math.round(r.left + r.width / 2);
- const cy = Math.round(r.top + r.height / 2);
- const topmost = _shadowAwareElementFromPoint(cx, cy);
+ let cx = Math.round(r.left + r.width / 2);
+ let cy = Math.round(r.top + r.height / 2);
+ let topmost = _shadowAwareElementFromPoint(cx, cy);
+
+ // If covered at center, test if scrolling can clear a fixed/sticky header
+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {
+ const bRect = topmost.getBoundingClientRect();
+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {
+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
+ r = el.getBoundingClientRect();
+ cx = Math.round(r.left + r.width / 2);
+ cy = Math.round(r.top + r.height / 2);
+ topmost = _shadowAwareElementFromPoint(cx, cy);
+ }
+ }
+
+ // If still covered at center, test sample perimeter points
+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {
+ const sampleOffsets = [
+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],
+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],
+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ ];
+ for (const [sx, sy] of sampleOffsets) {
+ if (sx < 0 || sy < 0 || sx > window.innerWidth || sy > window.innerHeight) continue;
+ const sampleHit = _shadowAwareElementFromPoint(sx, sy);
+ if (sampleHit && (_isComposedAncestor(el, sampleHit) || _isComposedAncestor(sampleHit, el))) {
+ topmost = sampleHit;
+ cx = sx;
+ cy = sy;
+ break;
+ }
+ }
+ }
+
+ // If still covered, check if the blocker chain has pointer-events: none
+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {
+ try {
+ const elements = document.elementsFromPoint ? document.elementsFromPoint(cx, cy) : [];
+ const targetIdx = elements.indexOf(el);
+ if (targetIdx > 0) {
+ const allAboveNone = elements.slice(0, targetIdx).every(item => {
+ const pe = window.getComputedStyle(item).pointerEvents;
+ return pe === 'none' || _isComposedAncestor(el, item) || _isComposedAncestor(item, el);
+ });
+ if (allAboveNone) topmost = el;
+ }
+ } catch {}
+ }
+
if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {
// Another element is painted on top. Give the model actionable
// info (what's blocking, where, what to do) instead of silently
@@ -6636,7 +6755,7 @@
);
}
if (!_isFullyVisibleForInteraction(el)) {
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
}
try { el.focus({ preventScroll: true }); } catch {}
const rect = el.getBoundingClientRect();
@@ -7020,7 +7139,7 @@
return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7168,7 +7287,7 @@
return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids — do NOT guess ref numbers or invent placeholders.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7355,7 +7474,7 @@
return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids — do NOT guess ref numbers or invent placeholders.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -7791,7 +7910,7 @@
: '';
return { success: false, error: `ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids.`, suggestions };
}
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
showAgentWorkingTarget(el, 'ax_resolve_rect');
const r = el.getBoundingClientRect();
const cx = r.left + r.width / 2;
@@ -7812,11 +7931,51 @@
hasStatefulSemantics,
} = fallbackStatic;
let topmost = null;
- try { if (forClickFallback && inViewport) topmost = document.elementFromPoint(cx, cy); } catch {}
- const hitOk = !forClickFallback || (!!topmost && (
+ let effectiveCx = cx;
+ let effectiveCy = cy;
+ if (forClickFallback && inViewport) {
+ try { topmost = document.elementFromPoint(cx, cy); } catch {}
+ }
+ let hitOk = !forClickFallback || (!!topmost && (
topmost === el
|| el.contains?.(topmost)
+ || topmost.contains?.(el)
));
+ if (forClickFallback && inViewport && !hitOk) {
+ const sampleOffsets = [
+ [cx, r.top + Math.max(2, Math.min(8, r.height * 0.2))],
+ [cx, r.bottom - Math.max(2, Math.min(8, r.height * 0.2))],
+ [r.left + Math.max(2, Math.min(8, r.width * 0.2)), cy],
+ [r.right - Math.max(2, Math.min(8, r.width * 0.2)), cy],
+ ];
+ for (const [sx, sy] of sampleOffsets) {
+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
+ let sampleHit = null;
+ try { sampleHit = document.elementFromPoint(sx, sy); } catch {}
+ if (sampleHit && (sampleHit === el || el.contains?.(sampleHit) || sampleHit.contains?.(el))) {
+ topmost = sampleHit;
+ effectiveCx = sx;
+ effectiveCy = sy;
+ hitOk = true;
+ break;
+ }
+ }
+ if (!hitOk && topmost) {
+ try {
+ const bRect = topmost.getBoundingClientRect();
+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < vh * 0.45) {
+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
+ const rNew = el.getBoundingClientRect();
+ effectiveCx = rNew.left + rNew.width / 2;
+ effectiveCy = rNew.top + rNew.height / 2;
+ topmost = document.elementFromPoint(effectiveCx, effectiveCy);
+ if (topmost && (topmost === el || el.contains?.(topmost) || topmost.contains?.(el))) {
+ hitOk = true;
+ }
+ }
+ } catch {}
+ }
+ }
const interactiveHitDescendant = forClickFallback
? _axInteractiveHitDescendant(el, topmost)
: null;
@@ -7839,8 +7998,8 @@
tag,
role,
name,
- x: Math.round(cx),
- y: Math.round(cy),
+ x: Math.round(effectiveCx),
+ y: Math.round(effectiveCy),
rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },
inViewport,
...(forClickFallback ? {
diff --git a/src/firefox/src/agent/loop-detector.js b/src/firefox/src/agent/loop-detector.js
index 44aa69cab..bd93586ca 100644
--- a/src/firefox/src/agent/loop-detector.js
+++ b/src/firefox/src/agent/loop-detector.js
@@ -621,21 +621,28 @@ export class LoopDetector {
}
const failures = this.failedActionLoops.get(tabId) || new Map();
if (this._isToolResultErroredForLoop(toolName, toolArgs, toolResult)) {
+ const isOcclusionFailure = toolResult?.occluded === true
+ || /occluded|covered by another element|topmost/i.test(String(toolResult?.error || ''));
+ const maxAttempts = isOcclusionFailure ? 4 : 3;
const attempts = (failures.get(failureScope) || 0) + 1;
failures.set(failureScope, attempts);
if (failures.size > 32) failures.delete(failures.keys().next().value);
this.failedActionLoops.set(tabId, failures);
- if (attempts >= 3) {
+ if (attempts >= maxAttempts) {
this._clearLoopState(tabId);
return {
kind: 'stop',
- message: `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,
+ message: isOcclusionFailure
+ ? `Stopped: ${toolName} failed ${maxAttempts} times because the target is occluded by an overlapping layer or docked header. Close the overlay, scroll past the obstruction, or re-read the page before retrying.`
+ : `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,
};
}
- if (attempts === 2) {
+ if (attempts >= 2) {
return {
kind: 'nudge',
- warning: `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,
+ warning: isOcclusionFailure
+ ? `[OVERLAY OCCLUSION: ${toolName} failed because the target element is occluded or covered by an overlapping element (such as a modal backdrop, fixed header, or sticky toolbar). Try closing the overlay (e.g. clicking a close/dismiss button or pressing Escape), scrolling the element into clear view, or re-reading the accessibility tree.]`
+ : `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,
};
}
} else if (toolResult?.success === true && toolResult?.verified !== false) {
diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.js
index c85f30c57..010ab2ba5 100644
--- a/src/firefox/src/content/accessibility-tree.js
+++ b/src/firefox/src/content/accessibility-tree.js
@@ -598,6 +598,58 @@
return hit;
}
+ function isNodeOccluded(el) {
+ if (!el || !el.isConnected || el.nodeType !== Node.ELEMENT_NODE) return false;
+ try {
+ const r = el.getBoundingClientRect();
+ if (r.width < 1 || r.height < 1) return false;
+ const vw = window.innerWidth || 800;
+ const vh = window.innerHeight || 600;
+ if (r.top >= vh || r.bottom <= 0 || r.left >= vw || r.right <= 0) return false;
+
+ const cx = r.left + r.width / 2;
+ const cy = r.top + r.height / 2;
+ if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
+ const hit = deepestOpenShadowHit(cx, cy);
+ if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;
+ }
+
+ const sampleOffsets = [
+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],
+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],
+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ ];
+ for (const [sx, sy] of sampleOffsets) {
+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;
+ const sampleHit = deepestOpenShadowHit(sx, sy);
+ if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {
+ return false;
+ }
+ }
+
+ if (document.elementsFromPoint && cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {
+ const elements = document.elementsFromPoint(cx, cy);
+ const targetIdx = elements.indexOf(el);
+ if (targetIdx > 0) {
+ const allAboveNone = elements.slice(0, targetIdx).every(item => {
+ try {
+ const pe = window.getComputedStyle(item).pointerEvents;
+ return pe === 'none' || el.contains(item) || item.contains(el);
+ } catch {
+ return false;
+ }
+ });
+ if (allAboveNone) return false;
+ }
+ }
+
+ return true;
+ } catch {
+ return false;
+ }
+ }
+
function visualTargetEligibility(el) {
const tag = el.tagName?.toLowerCase() || '';
if (tag === 'button') return 'semantic-button';
@@ -879,6 +931,11 @@
}
} catch {}
}
+ try {
+ if (isInteractive(el) && isNodeOccluded(el)) {
+ line += ' occluded=true';
+ }
+ } catch {}
if (inputType === 'checkbox' || inputType === 'radio') {
line += ` checked=${el.checked ? 'true' : 'false'}`;
} else if (['checkbox', 'radio', 'switch'].includes(attrRole) || el.hasAttribute('aria-checked')) {
@@ -1565,6 +1622,13 @@
'[aria-modal="true"]',
'[role=combobox][aria-expanded="true"]',
'dialog[open]',
+ '[data-overlay]',
+ '.modal.show',
+ '.modal-overlay',
+ '[class*="modal"][class*="open"]',
+ '[class*="DialogContent"]',
+ '[class*="ModalContent"]',
+ '[data-state="open"][role="dialog"]',
];
const overlayEls = [];
const seen = new WeakSet();
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 06bf530be..084bed2ec 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -520,18 +520,89 @@
: null;
}
+ function _getViewportDockedInsets(view = window) {
+ let top = 0;
+ let bottom = 0;
+ try {
+ const doc = view.document;
+ if (!doc) return { top: 0, bottom: 0 };
+ const htmlStyle = view.getComputedStyle(doc.documentElement);
+ const bodyStyle = doc.body ? view.getComputedStyle(doc.body) : null;
+ const parsePadding = val => {
+ const n = parseFloat(val);
+ return Number.isFinite(n) && n > 0 ? n : 0;
+ };
+ top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
+ bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
+ if (bodyStyle) {
+ top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
+ bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
+ }
+ const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');
+ const vw = view.innerWidth || 800;
+ const vh = view.innerHeight || 600;
+ for (const c of candidates) {
+ if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;
+ const cs = view.getComputedStyle(c);
+ if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
+ if (cs.display === 'none' || cs.visibility === 'hidden') continue;
+ const r = c.getBoundingClientRect();
+ if (r.width < vw * 0.4) continue;
+ if (r.top <= 10 && r.bottom > 0 && r.bottom < vh * 0.4) {
+ if (r.bottom > top) top = r.bottom;
+ } else if (r.bottom >= vh - 10 && r.top < vh && r.top > vh * 0.6) {
+ const h = vh - r.top;
+ if (h > bottom) bottom = h;
+ }
+ }
+ } catch {}
+ return {
+ top: Math.min(top, (view.innerHeight || 600) * 0.4),
+ bottom: Math.min(bottom, (view.innerHeight || 600) * 0.4),
+ };
+ }
+
+ function _scrollElementIntoClearView(el) {
+ if (!el?.isConnected) return;
+ try {
+ const view = el.ownerDocument?.defaultView || window;
+ const insets = _getViewportDockedInsets(view);
+ const rBefore = el.getBoundingClientRect();
+ const inClear = (
+ rBefore.width >= 1
+ && rBefore.height >= 1
+ && rBefore.left >= 0
+ && rBefore.right <= view.innerWidth
+ && rBefore.top >= insets.top
+ && rBefore.bottom <= (view.innerHeight - insets.bottom)
+ );
+ if (!inClear) {
+ el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });
+ const rAfter = el.getBoundingClientRect();
+ if (rAfter.top < insets.top) {
+ view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });
+ } else if (rAfter.bottom > view.innerHeight - insets.bottom) {
+ view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });
+ }
+ }
+ } catch {
+ try { el.scrollIntoView({ block: 'center', inline: 'nearest' }); } catch {}
+ }
+ }
+
function _isFullyVisibleForInteraction(el) {
try {
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
+ const insets = _getViewportDockedInsets(view);
if (
rect.width < 1
|| rect.height < 1
|| rect.left < 0
- || rect.top < 0
+ || rect.top < insets.top
|| rect.right > view.innerWidth
- || rect.bottom > view.innerHeight
+ || rect.bottom > (view.innerHeight - insets.bottom)
) return false;
for (let node = _composedParent(el); node && node !== el.ownerDocument; node = _composedParent(node)) {
if (node.nodeType !== Node.ELEMENT_NODE) continue;
@@ -2144,9 +2215,7 @@
// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)
if (el.tagName !== 'SELECT') {
if (actionDeadlineExpired()) return deadlineFailure();
- // BiDi validates the target in the same turn. A smooth scroll leaves a
- // transient offscreen geometry window where that validation must fail.
- el.scrollIntoView({ behavior: params._bidiPrepare ? 'instant' : 'smooth', block: 'center' });
+ _scrollElementIntoClearView(el);
}
// Occlusion hit-test: for text/selector/index clicks, verify that the
@@ -2157,11 +2226,59 @@
// SELECT (already handled).
if (el.tagName !== 'SELECT' && params.x == null && params.y == null) {
try {
- const r = el.getBoundingClientRect();
+ let r = el.getBoundingClientRect();
if (r.width >= 1 && r.height >= 1 && r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth) {
- const cx = Math.round(r.left + r.width / 2);
- const cy = Math.round(r.top + r.height / 2);
- const topmost = _shadowAwareElementFromPoint(cx, cy);
+ let cx = Math.round(r.left + r.width / 2);
+ let cy = Math.round(r.top + r.height / 2);
+ let topmost = _shadowAwareElementFromPoint(cx, cy);
+
+ // If covered at center, test if scrolling can clear a fixed/sticky header
+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {
+ const bRect = topmost.getBoundingClientRect();
+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {
+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
+ r = el.getBoundingClientRect();
+ cx = Math.round(r.left + r.width / 2);
+ cy = Math.round(r.top + r.height / 2);
+ topmost = _shadowAwareElementFromPoint(cx, cy);
+ }
+ }
+
+ // If still covered at center, test sample perimeter points
+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {
+ const sampleOffsets = [
+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],
+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],
+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],
+ ];
+ for (const [sx, sy] of sampleOffsets) {
+ if (sx < 0 || sy < 0 || sx > window.innerWidth || sy > window.innerHeight) continue;
+ const sampleHit = _shadowAwareElementFromPoint(sx, sy);
+ if (sampleHit && _hitTestMatchesTarget(el, sampleHit)) {
+ topmost = sampleHit;
+ cx = sx;
+ cy = sy;
+ break;
+ }
+ }
+ }
+
+ // If still covered, check if the blocker chain has pointer-events: none
+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {
+ try {
+ const elements = document.elementsFromPoint ? document.elementsFromPoint(cx, cy) : [];
+ const targetIdx = elements.indexOf(el);
+ if (targetIdx > 0) {
+ const allAboveNone = elements.slice(0, targetIdx).every(item => {
+ const pe = window.getComputedStyle(item).pointerEvents;
+ return pe === 'none' || _hitTestMatchesTarget(el, item);
+ });
+ if (allAboveNone) topmost = el;
+ }
+ } catch {}
+ }
+
if (topmost && !_hitTestMatchesTarget(el, topmost)) {
let blockerInfo = topmost.tagName.toLowerCase();
const role = topmost.getAttribute && topmost.getAttribute('role');
@@ -5730,11 +5847,7 @@
const targetName = canonicalTargetName || _axAccessibleName(el);
if (!_isFullyVisibleForInteraction(el)) {
try {
- el.scrollIntoView({
- block: 'center',
- inline: 'center',
- ...(msg.params?._bidiPrepare ? { behavior: 'instant' } : {}),
- });
+ _scrollElementIntoClearView(el);
} catch {}
}
try { el.focus({ preventScroll: true }); } catch {}
@@ -6058,7 +6171,7 @@
return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -6176,7 +6289,7 @@
return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree to get fresh ids.`, { suggestions });
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -6350,7 +6463,7 @@
const el = window.__wb_ax_lookup(ref_id);
if (!el) return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree.`);
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
@@ -6736,7 +6849,7 @@
return { success: false, error: `ref_id ${ref_id} not found.`, suggestions };
}
if (actionDeadlineExpired()) return deadlineFailure();
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try { _scrollElementIntoClearView(el); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
const r = el.getBoundingClientRect();
const cx = r.left + r.width / 2;