Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: handle overlapping content elements and docked headers (#3085)
Emre Sokullubda9ae760b5c
diff --git a/src/chrome/src/agent/loop-detector.js b/src/chrome/src/agent/loop-detector.jsindex 44aa69cab..bd93586ca 100644--- a/src/chrome/src/agent/loop-detector.js+++ b/src/chrome/src/agent/loop-detector.js@@ -621,21 +621,28 @@ export class LoopDetector {}const failures = this.failedActionLoops.get(tabId) || new Map();if (this._isToolResultErroredForLoop(toolName, toolArgs, toolResult)) {+ const isOcclusionFailure = toolResult?.occluded === true+ || /occluded|covered by another element|topmost/i.test(String(toolResult?.error || ''));+ const maxAttempts = isOcclusionFailure ? 4 : 3;const attempts = (failures.get(failureScope) || 0) + 1;failures.set(failureScope, attempts);if (failures.size > 32) failures.delete(failures.keys().next().value);this.failedActionLoops.set(tabId, failures);- if (attempts >= 3) {+ if (attempts >= maxAttempts) {this._clearLoopState(tabId);return {kind: 'stop',- message: `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,+ message: isOcclusionFailure+ ? `Stopped: ${toolName} failed ${maxAttempts} times because the target is occluded by an overlapping layer or docked header. Close the overlay, scroll past the obstruction, or re-read the page before retrying.`+ : `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,};}- if (attempts === 2) {+ if (attempts >= 2) {return {kind: 'nudge',- warning: `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,+ warning: isOcclusionFailure+ ? `[OVERLAY OCCLUSION: ${toolName} failed because the target element is occluded or covered by an overlapping element (such as a modal backdrop, fixed header, or sticky toolbar). Try closing the overlay (e.g. clicking a close/dismiss button or pressing Escape), scrolling the element into clear view, or re-reading the accessibility tree.]`+ : `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,};}} else if (toolResult?.success === true && toolResult?.verified !== false) {diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.jsindex c85f30c57..010ab2ba5 100644--- a/src/chrome/src/content/accessibility-tree.js+++ b/src/chrome/src/content/accessibility-tree.js@@ -598,6 +598,58 @@return hit;}+ function isNodeOccluded(el) {+ if (!el || !el.isConnected || el.nodeType !== Node.ELEMENT_NODE) return false;+ try {+ const r = el.getBoundingClientRect();+ if (r.width < 1 || r.height < 1) return false;+ const vw = window.innerWidth || 800;+ const vh = window.innerHeight || 600;+ if (r.top >= vh || r.bottom <= 0 || r.left >= vw || r.right <= 0) return false;++ const cx = r.left + r.width / 2;+ const cy = r.top + r.height / 2;+ if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {+ const hit = deepestOpenShadowHit(cx, cy);+ if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;+ }++ const sampleOffsets = [+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],+ ];+ for (const [sx, sy] of sampleOffsets) {+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;+ const sampleHit = deepestOpenShadowHit(sx, sy);+ if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {+ return false;+ }+ }++ if (document.elementsFromPoint && cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {+ const elements = document.elementsFromPoint(cx, cy);+ const targetIdx = elements.indexOf(el);+ if (targetIdx > 0) {+ const allAboveNone = elements.slice(0, targetIdx).every(item => {+ try {+ const pe = window.getComputedStyle(item).pointerEvents;+ return pe === 'none' || el.contains(item) || item.contains(el);+ } catch {+ return false;+ }+ });+ if (allAboveNone) return false;+ }+ }++ return true;+ } catch {+ return false;+ }+ }+function visualTargetEligibility(el) {const tag = el.tagName?.toLowerCase() || '';if (tag === 'button') return 'semantic-button';@@ -879,6 +931,11 @@}} catch {}}+ try {+ if (isInteractive(el) && isNodeOccluded(el)) {+ line += ' occluded=true';+ }+ } catch {}if (inputType === 'checkbox' || inputType === 'radio') {line += ` checked=${el.checked ? 'true' : 'false'}`;} else if (['checkbox', 'radio', 'switch'].includes(attrRole) || el.hasAttribute('aria-checked')) {@@ -1565,6 +1622,13 @@'[aria-modal="true"]','[role=combobox][aria-expanded="true"]','dialog[open]',+ '[data-overlay]',+ '.modal.show',+ '.modal-overlay',+ '[class*="modal"][class*="open"]',+ '[class*="DialogContent"]',+ '[class*="ModalContent"]',+ '[data-state="open"][role="dialog"]',];const overlayEls = [];const seen = new WeakSet();diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 82d28f8ff..1c08826ae 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -311,18 +311,89 @@: null;}+ function _getViewportDockedInsets(view = window) {+ let top = 0;+ let bottom = 0;+ try {+ const doc = view.document;+ if (!doc) return { top: 0, bottom: 0 };+ const htmlStyle = view.getComputedStyle(doc.documentElement);+ const bodyStyle = doc.body ? view.getComputedStyle(doc.body) : null;+ const parsePadding = val => {+ const n = parseFloat(val);+ return Number.isFinite(n) && n > 0 ? n : 0;+ };+ top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));+ bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));+ if (bodyStyle) {+ top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));+ bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));+ }+ const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');+ const vw = view.innerWidth || 800;+ const vh = view.innerHeight || 600;+ for (const c of candidates) {+ if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;+ const cs = view.getComputedStyle(c);+ if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;+ if (cs.display === 'none' || cs.visibility === 'hidden') continue;+ const r = c.getBoundingClientRect();+ if (r.width < vw * 0.4) continue;+ if (r.top <= 10 && r.bottom > 0 && r.bottom < vh * 0.4) {+ if (r.bottom > top) top = r.bottom;+ } else if (r.bottom >= vh - 10 && r.top < vh && r.top > vh * 0.6) {+ const h = vh - r.top;+ if (h > bottom) bottom = h;+ }+ }+ } catch {}+ return {+ top: Math.min(top, (view.innerHeight || 600) * 0.4),+ bottom: Math.min(bottom, (view.innerHeight || 600) * 0.4),+ };+ }++ function _scrollElementIntoClearView(el) {+ if (!el?.isConnected) return;+ try {+ const view = el.ownerDocument?.defaultView || window;+ const insets = _getViewportDockedInsets(view);+ const rBefore = el.getBoundingClientRect();+ const inClear = (+ rBefore.width >= 1+ && rBefore.height >= 1+ && rBefore.left >= 0+ && rBefore.right <= view.innerWidth+ && rBefore.top >= insets.top+ && rBefore.bottom <= (view.innerHeight - insets.bottom)+ );+ if (!inClear) {+ el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });+ const rAfter = el.getBoundingClientRect();+ if (rAfter.top < insets.top) {+ view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });+ } else if (rAfter.bottom > view.innerHeight - insets.bottom) {+ view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });+ }+ }+ } catch {+ try { el.scrollIntoView({ block: 'center', inline: 'nearest' }); } catch {}+ }+ }+function _isFullyVisibleForInteraction(el) {try {if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();+ const insets = _getViewportDockedInsets(view);if (rect.width < 1|| rect.height < 1|| rect.left < 0- || rect.top < 0+ || rect.top < insets.top|| rect.right > view.innerWidth- || rect.bottom > view.innerHeight+ || rect.bottom > (view.innerHeight - insets.bottom)) return false;for (let node = _composedParent(el); node && node !== el.ownerDocument; node = _composedParent(node)) {if (node.nodeType !== Node.ELEMENT_NODE) continue;@@ -1893,7 +1964,7 @@// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)if (el.tagName !== 'SELECT') {if (actionDeadlineExpired()) return deadlineFailure();- el.scrollIntoView({ behavior: 'smooth', block: 'center' });+ _scrollElementIntoClearView(el);}// Occlusion hit-test: for text/selector/index clicks, verify that the@@ -1906,11 +1977,59 @@// elements with real bounding rects.if (el.tagName !== 'SELECT' && params.x == null && params.y == null) {try {- const r = el.getBoundingClientRect();+ let r = el.getBoundingClientRect();if (r.width >= 1 && r.height >= 1 && r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth) {- const cx = Math.round(r.left + r.width / 2);- const cy = Math.round(r.top + r.height / 2);- const topmost = _shadowAwareElementFromPoint(cx, cy);+ let cx = Math.round(r.left + r.width / 2);+ let cy = Math.round(r.top + r.height / 2);+ let topmost = _shadowAwareElementFromPoint(cx, cy);++ // If covered at center, test if scrolling can clear a fixed/sticky header+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {+ const bRect = topmost.getBoundingClientRect();+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });+ r = el.getBoundingClientRect();+ cx = Math.round(r.left + r.width / 2);+ cy = Math.round(r.top + r.height / 2);+ topmost = _shadowAwareElementFromPoint(cx, cy);+ }+ }++ // If still covered at center, test sample perimeter points+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {+ const sampleOffsets = [+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],+ ];+ for (const [sx, sy] of sampleOffsets) {+ if (sx < 0 || sy < 0 || sx > window.innerWidth || sy > window.innerHeight) continue;+ const sampleHit = _shadowAwareElementFromPoint(sx, sy);+ if (sampleHit && (_isComposedAncestor(el, sampleHit) || _isComposedAncestor(sampleHit, el))) {+ topmost = sampleHit;+ cx = sx;+ cy = sy;+ break;+ }+ }+ }++ // If still covered, check if the blocker chain has pointer-events: none+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {+ try {+ const elements = document.elementsFromPoint ? document.elementsFromPoint(cx, cy) : [];+ const targetIdx = elements.indexOf(el);+ if (targetIdx > 0) {+ const allAboveNone = elements.slice(0, targetIdx).every(item => {+ const pe = window.getComputedStyle(item).pointerEvents;+ return pe === 'none' || _isComposedAncestor(el, item) || _isComposedAncestor(item, el);+ });+ if (allAboveNone) topmost = el;+ }+ } catch {}+ }+if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {// Another element is painted on top. Give the model actionable// info (what's blocking, where, what to do) instead of silently@@ -6636,7 +6755,7 @@);}if (!_isFullyVisibleForInteraction(el)) {- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}}try { el.focus({ preventScroll: true }); } catch {}const rect = el.getBoundingClientRect();@@ -7020,7 +7139,7 @@return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -7168,7 +7287,7 @@return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids — do NOT guess ref numbers or invent placeholders.`, { suggestions });}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -7355,7 +7474,7 @@return failure(`ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids — do NOT guess ref numbers or invent placeholders.`, { suggestions });}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -7791,7 +7910,7 @@: '';return { success: false, error: `ref_id ${ref_id} not found.${formatNote} The element may have been removed or the page replaced.${hint} Re-read the accessibility tree to get fresh ids.`, suggestions };}- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}showAgentWorkingTarget(el, 'ax_resolve_rect');const r = el.getBoundingClientRect();const cx = r.left + r.width / 2;@@ -7812,11 +7931,51 @@hasStatefulSemantics,} = fallbackStatic;let topmost = null;- try { if (forClickFallback && inViewport) topmost = document.elementFromPoint(cx, cy); } catch {}- const hitOk = !forClickFallback || (!!topmost && (+ let effectiveCx = cx;+ let effectiveCy = cy;+ if (forClickFallback && inViewport) {+ try { topmost = document.elementFromPoint(cx, cy); } catch {}+ }+ let hitOk = !forClickFallback || (!!topmost && (topmost === el|| el.contains?.(topmost)+ || topmost.contains?.(el)));+ if (forClickFallback && inViewport && !hitOk) {+ const sampleOffsets = [+ [cx, r.top + Math.max(2, Math.min(8, r.height * 0.2))],+ [cx, r.bottom - Math.max(2, Math.min(8, r.height * 0.2))],+ [r.left + Math.max(2, Math.min(8, r.width * 0.2)), cy],+ [r.right - Math.max(2, Math.min(8, r.width * 0.2)), cy],+ ];+ for (const [sx, sy] of sampleOffsets) {+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;+ let sampleHit = null;+ try { sampleHit = document.elementFromPoint(sx, sy); } catch {}+ if (sampleHit && (sampleHit === el || el.contains?.(sampleHit) || sampleHit.contains?.(el))) {+ topmost = sampleHit;+ effectiveCx = sx;+ effectiveCy = sy;+ hitOk = true;+ break;+ }+ }+ if (!hitOk && topmost) {+ try {+ const bRect = topmost.getBoundingClientRect();+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < vh * 0.45) {+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });+ const rNew = el.getBoundingClientRect();+ effectiveCx = rNew.left + rNew.width / 2;+ effectiveCy = rNew.top + rNew.height / 2;+ topmost = document.elementFromPoint(effectiveCx, effectiveCy);+ if (topmost && (topmost === el || el.contains?.(topmost) || topmost.contains?.(el))) {+ hitOk = true;+ }+ }+ } catch {}+ }+ }const interactiveHitDescendant = forClickFallback? _axInteractiveHitDescendant(el, topmost): null;@@ -7839,8 +7998,8 @@tag,role,name,- x: Math.round(cx),- y: Math.round(cy),+ x: Math.round(effectiveCx),+ y: Math.round(effectiveCy),rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },inViewport,...(forClickFallback ? {diff --git a/src/firefox/src/agent/loop-detector.js b/src/firefox/src/agent/loop-detector.jsindex 44aa69cab..bd93586ca 100644--- a/src/firefox/src/agent/loop-detector.js+++ b/src/firefox/src/agent/loop-detector.js@@ -621,21 +621,28 @@ export class LoopDetector {}const failures = this.failedActionLoops.get(tabId) || new Map();if (this._isToolResultErroredForLoop(toolName, toolArgs, toolResult)) {+ const isOcclusionFailure = toolResult?.occluded === true+ || /occluded|covered by another element|topmost/i.test(String(toolResult?.error || ''));+ const maxAttempts = isOcclusionFailure ? 4 : 3;const attempts = (failures.get(failureScope) || 0) + 1;failures.set(failureScope, attempts);if (failures.size > 32) failures.delete(failures.keys().next().value);this.failedActionLoops.set(tabId, failures);- if (attempts >= 3) {+ if (attempts >= maxAttempts) {this._clearLoopState(tabId);return {kind: 'stop',- message: `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,+ message: isOcclusionFailure+ ? `Stopped: ${toolName} failed ${maxAttempts} times because the target is occluded by an overlapping layer or docked header. Close the overlay, scroll past the obstruction, or re-read the page before retrying.`+ : `Stopped: ${toolName} failed or made no progress three times for the same target. Repeating it or switching to a precomputed fallback cannot make progress without fresh page evidence.`,};}- if (attempts === 2) {+ if (attempts >= 2) {return {kind: 'nudge',- warning: `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,+ warning: isOcclusionFailure+ ? `[OVERLAY OCCLUSION: ${toolName} failed because the target element is occluded or covered by an overlapping element (such as a modal backdrop, fixed header, or sticky toolbar). Try closing the overlay (e.g. clicking a close/dismiss button or pressing Escape), scrolling the element into clear view, or re-reading the accessibility tree.]`+ : `[FAILED ACTION LOOP: ${toolName} has failed or made no progress twice for the same target. Do not retry it or use a queued fallback. Re-read the page/tree and choose a new action from current evidence.]`,};}} else if (toolResult?.success === true && toolResult?.verified !== false) {diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.jsindex c85f30c57..010ab2ba5 100644--- a/src/firefox/src/content/accessibility-tree.js+++ b/src/firefox/src/content/accessibility-tree.js@@ -598,6 +598,58 @@return hit;}+ function isNodeOccluded(el) {+ if (!el || !el.isConnected || el.nodeType !== Node.ELEMENT_NODE) return false;+ try {+ const r = el.getBoundingClientRect();+ if (r.width < 1 || r.height < 1) return false;+ const vw = window.innerWidth || 800;+ const vh = window.innerHeight || 600;+ if (r.top >= vh || r.bottom <= 0 || r.left >= vw || r.right <= 0) return false;++ const cx = r.left + r.width / 2;+ const cy = r.top + r.height / 2;+ if (cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {+ const hit = deepestOpenShadowHit(cx, cy);+ if (hit && (hit === el || el.contains(hit) || hit.contains(el))) return false;+ }++ const sampleOffsets = [+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],+ ];+ for (const [sx, sy] of sampleOffsets) {+ if (sx < 0 || sy < 0 || sx > vw || sy > vh) continue;+ const sampleHit = deepestOpenShadowHit(sx, sy);+ if (sampleHit && (sampleHit === el || el.contains(sampleHit) || sampleHit.contains(el))) {+ return false;+ }+ }++ if (document.elementsFromPoint && cx >= 0 && cx <= vw && cy >= 0 && cy <= vh) {+ const elements = document.elementsFromPoint(cx, cy);+ const targetIdx = elements.indexOf(el);+ if (targetIdx > 0) {+ const allAboveNone = elements.slice(0, targetIdx).every(item => {+ try {+ const pe = window.getComputedStyle(item).pointerEvents;+ return pe === 'none' || el.contains(item) || item.contains(el);+ } catch {+ return false;+ }+ });+ if (allAboveNone) return false;+ }+ }++ return true;+ } catch {+ return false;+ }+ }+function visualTargetEligibility(el) {const tag = el.tagName?.toLowerCase() || '';if (tag === 'button') return 'semantic-button';@@ -879,6 +931,11 @@}} catch {}}+ try {+ if (isInteractive(el) && isNodeOccluded(el)) {+ line += ' occluded=true';+ }+ } catch {}if (inputType === 'checkbox' || inputType === 'radio') {line += ` checked=${el.checked ? 'true' : 'false'}`;} else if (['checkbox', 'radio', 'switch'].includes(attrRole) || el.hasAttribute('aria-checked')) {@@ -1565,6 +1622,13 @@'[aria-modal="true"]','[role=combobox][aria-expanded="true"]','dialog[open]',+ '[data-overlay]',+ '.modal.show',+ '.modal-overlay',+ '[class*="modal"][class*="open"]',+ '[class*="DialogContent"]',+ '[class*="ModalContent"]',+ '[data-state="open"][role="dialog"]',];const overlayEls = [];const seen = new WeakSet();diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 06bf530be..084bed2ec 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -520,18 +520,89 @@: null;}+ function _getViewportDockedInsets(view = window) {+ let top = 0;+ let bottom = 0;+ try {+ const doc = view.document;+ if (!doc) return { top: 0, bottom: 0 };+ const htmlStyle = view.getComputedStyle(doc.documentElement);+ const bodyStyle = doc.body ? view.getComputedStyle(doc.body) : null;+ const parsePadding = val => {+ const n = parseFloat(val);+ return Number.isFinite(n) && n > 0 ? n : 0;+ };+ top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));+ bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));+ if (bodyStyle) {+ top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));+ bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));+ }+ const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');+ const vw = view.innerWidth || 800;+ const vh = view.innerHeight || 600;+ for (const c of candidates) {+ if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;+ const cs = view.getComputedStyle(c);+ if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;+ if (cs.display === 'none' || cs.visibility === 'hidden') continue;+ const r = c.getBoundingClientRect();+ if (r.width < vw * 0.4) continue;+ if (r.top <= 10 && r.bottom > 0 && r.bottom < vh * 0.4) {+ if (r.bottom > top) top = r.bottom;+ } else if (r.bottom >= vh - 10 && r.top < vh && r.top > vh * 0.6) {+ const h = vh - r.top;+ if (h > bottom) bottom = h;+ }+ }+ } catch {}+ return {+ top: Math.min(top, (view.innerHeight || 600) * 0.4),+ bottom: Math.min(bottom, (view.innerHeight || 600) * 0.4),+ };+ }++ function _scrollElementIntoClearView(el) {+ if (!el?.isConnected) return;+ try {+ const view = el.ownerDocument?.defaultView || window;+ const insets = _getViewportDockedInsets(view);+ const rBefore = el.getBoundingClientRect();+ const inClear = (+ rBefore.width >= 1+ && rBefore.height >= 1+ && rBefore.left >= 0+ && rBefore.right <= view.innerWidth+ && rBefore.top >= insets.top+ && rBefore.bottom <= (view.innerHeight - insets.bottom)+ );+ if (!inClear) {+ el.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' });+ const rAfter = el.getBoundingClientRect();+ if (rAfter.top < insets.top) {+ view.scrollBy({ top: rAfter.top - insets.top - 16, behavior: 'instant' });+ } else if (rAfter.bottom > view.innerHeight - insets.bottom) {+ view.scrollBy({ top: rAfter.bottom - (view.innerHeight - insets.bottom) + 16, behavior: 'instant' });+ }+ }+ } catch {+ try { el.scrollIntoView({ block: 'center', inline: 'nearest' }); } catch {}+ }+ }+function _isFullyVisibleForInteraction(el) {try {if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();+ const insets = _getViewportDockedInsets(view);if (rect.width < 1|| rect.height < 1|| rect.left < 0- || rect.top < 0+ || rect.top < insets.top|| rect.right > view.innerWidth- || rect.bottom > view.innerHeight+ || rect.bottom > (view.innerHeight - insets.bottom)) return false;for (let node = _composedParent(el); node && node !== el.ownerDocument; node = _composedParent(node)) {if (node.nodeType !== Node.ELEMENT_NODE) continue;@@ -2144,9 +2215,7 @@// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)if (el.tagName !== 'SELECT') {if (actionDeadlineExpired()) return deadlineFailure();- // BiDi validates the target in the same turn. A smooth scroll leaves a- // transient offscreen geometry window where that validation must fail.- el.scrollIntoView({ behavior: params._bidiPrepare ? 'instant' : 'smooth', block: 'center' });+ _scrollElementIntoClearView(el);}// Occlusion hit-test: for text/selector/index clicks, verify that the@@ -2157,11 +2226,59 @@// SELECT (already handled).if (el.tagName !== 'SELECT' && params.x == null && params.y == null) {try {- const r = el.getBoundingClientRect();+ let r = el.getBoundingClientRect();if (r.width >= 1 && r.height >= 1 && r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth) {- const cx = Math.round(r.left + r.width / 2);- const cy = Math.round(r.top + r.height / 2);- const topmost = _shadowAwareElementFromPoint(cx, cy);+ let cx = Math.round(r.left + r.width / 2);+ let cy = Math.round(r.top + r.height / 2);+ let topmost = _shadowAwareElementFromPoint(cx, cy);++ // If covered at center, test if scrolling can clear a fixed/sticky header+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {+ const bRect = topmost.getBoundingClientRect();+ if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {+ window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });+ r = el.getBoundingClientRect();+ cx = Math.round(r.left + r.width / 2);+ cy = Math.round(r.top + r.height / 2);+ topmost = _shadowAwareElementFromPoint(cx, cy);+ }+ }++ // If still covered at center, test sample perimeter points+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {+ const sampleOffsets = [+ [cx, Math.round(r.top + Math.max(2, Math.min(8, r.height * 0.2)))],+ [cx, Math.round(r.bottom - Math.max(2, Math.min(8, r.height * 0.2)))],+ [Math.round(r.left + Math.max(2, Math.min(8, r.width * 0.2))), cy],+ [Math.round(r.right - Math.max(2, Math.min(8, r.width * 0.2))), cy],+ ];+ for (const [sx, sy] of sampleOffsets) {+ if (sx < 0 || sy < 0 || sx > window.innerWidth || sy > window.innerHeight) continue;+ const sampleHit = _shadowAwareElementFromPoint(sx, sy);+ if (sampleHit && _hitTestMatchesTarget(el, sampleHit)) {+ topmost = sampleHit;+ cx = sx;+ cy = sy;+ break;+ }+ }+ }++ // If still covered, check if the blocker chain has pointer-events: none+ if (topmost && !_hitTestMatchesTarget(el, topmost)) {+ try {+ const elements = document.elementsFromPoint ? document.elementsFromPoint(cx, cy) : [];+ const targetIdx = elements.indexOf(el);+ if (targetIdx > 0) {+ const allAboveNone = elements.slice(0, targetIdx).every(item => {+ const pe = window.getComputedStyle(item).pointerEvents;+ return pe === 'none' || _hitTestMatchesTarget(el, item);+ });+ if (allAboveNone) topmost = el;+ }+ } catch {}+ }+if (topmost && !_hitTestMatchesTarget(el, topmost)) {let blockerInfo = topmost.tagName.toLowerCase();const role = topmost.getAttribute && topmost.getAttribute('role');@@ -5730,11 +5847,7 @@const targetName = canonicalTargetName || _axAccessibleName(el);if (!_isFullyVisibleForInteraction(el)) {try {- el.scrollIntoView({- block: 'center',- inline: 'center',- ...(msg.params?._bidiPrepare ? { behavior: 'instant' } : {}),- });+ _scrollElementIntoClearView(el);} catch {}}try { el.focus({ preventScroll: true }); } catch {}@@ -6058,7 +6171,7 @@return failure(`set_checked only supports native input[type="checkbox"] controls; ${ref_id} resolved to ${tag || 'unknown'}${inputType ? `[type="${inputType}"]` : ''}.`);}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -6176,7 +6289,7 @@return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree to get fresh ids.`, { suggestions });}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -6350,7 +6463,7 @@const el = window.__wb_ax_lookup(ref_id);if (!el) return failure(`ref_id ${ref_id} not found. Re-read the accessibility tree.`);if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();@@ -6736,7 +6849,7 @@return { success: false, error: `ref_id ${ref_id} not found.`, suggestions };}if (actionDeadlineExpired()) return deadlineFailure();- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try { _scrollElementIntoClearView(el); } catch {}if (actionDeadlineExpired()) return deadlineFailure();const r = el.getBoundingClientRect();const cx = r.left + r.width / 2;