Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: preserve docked controls and correct obstruction scroll
Barack Sokulluc06464e3a941
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 21f14df47..abaef38ee 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -312,7 +312,7 @@: null;}- function _getViewportDockedInsets(view = window) {+ function _getViewportDockedInsets(view = window, target = null) {let top = 0;let bottom = 0;try {@@ -324,16 +324,27 @@const n = parseFloat(val);return Number.isFinite(n) && n > 0 ? n : 0;};- top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));- bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));+ let targetInTopDock = false;+ let targetInBottomDock = false;+ for (let node = target; node && node !== doc; node = _composedParent(node)) {+ if (node.nodeType !== Node.ELEMENT_NODE) continue;+ const style = view.getComputedStyle(node);+ if (style.position !== 'fixed' && style.position !== 'sticky') continue;+ const rect = node.getBoundingClientRect();+ if (rect.top <= 10 && rect.bottom > 0 && rect.bottom < view.innerHeight * 0.4) targetInTopDock = true;+ if (rect.bottom >= view.innerHeight - 10 && rect.top > view.innerHeight * 0.6) targetInBottomDock = true;+ }+ if (!targetInTopDock) top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));if (bodyStyle) {- top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));- bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));+ if (!targetInTopDock) top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));}const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');const vw = view.innerWidth || 800;const vh = view.innerHeight || 600;for (const c of candidates) {+ if (target && _isComposedAncestor(c, target)) continue;if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;const cs = view.getComputedStyle(c);if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;@@ -358,7 +369,7 @@if (!el?.isConnected) return;try {const view = el.ownerDocument?.defaultView || window;- const insets = _getViewportDockedInsets(view);+ const insets = _getViewportDockedInsets(view, el);// A viewport-only rect check misses elements clipped by a scrollable// ancestor even when their bounding box is inside the viewport.if (!_isFullyVisibleForInteraction(el, insets)) {@@ -380,7 +391,7 @@if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();- const insets = dockedInsets || _getViewportDockedInsets(view);+ const insets = dockedInsets || _getViewportDockedInsets(view, el);if (rect.width < 1|| rect.height < 1@@ -1981,7 +1992,7 @@if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {const bRect = topmost.getBoundingClientRect();if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });r = el.getBoundingClientRect();cx = Math.round(r.left + r.width / 2);cy = Math.round(r.top + r.height / 2);@@ -7956,7 +7967,7 @@}try { _scrollElementIntoClearView(el); } catch {}showAgentWorkingTarget(el, 'ax_resolve_rect');- const r = el.getBoundingClientRect();+ let r = el.getBoundingClientRect();const cx = r.left + r.width / 2;const cy = r.top + r.height / 2;const vw = window.innerWidth, vh = window.innerHeight;@@ -8008,10 +8019,10 @@try {const bRect = topmost.getBoundingClientRect();if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < vh * 0.45) {- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });- const rNew = el.getBoundingClientRect();- effectiveCx = rNew.left + rNew.width / 2;- effectiveCy = rNew.top + rNew.height / 2;+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });+ r = el.getBoundingClientRect();+ effectiveCx = r.left + r.width / 2;+ effectiveCy = r.top + r.height / 2;topmost = document.elementFromPoint(effectiveCx, effectiveCy);if (topmost && (topmost === el || el.contains?.(topmost) || topmost.contains?.(el))) {hitOk = true;diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex ed00a3446..7f1374d22 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -571,7 +571,7 @@: null;}- function _getViewportDockedInsets(view = window) {+ function _getViewportDockedInsets(view = window, target = null) {let top = 0;let bottom = 0;try {@@ -583,16 +583,27 @@const n = parseFloat(val);return Number.isFinite(n) && n > 0 ? n : 0;};- top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));- bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));+ let targetInTopDock = false;+ let targetInBottomDock = false;+ for (let node = target; node && node !== doc; node = _composedParent(node)) {+ if (node.nodeType !== Node.ELEMENT_NODE) continue;+ const style = view.getComputedStyle(node);+ if (style.position !== 'fixed' && style.position !== 'sticky') continue;+ const rect = node.getBoundingClientRect();+ if (rect.top <= 10 && rect.bottom > 0 && rect.bottom < view.innerHeight * 0.4) targetInTopDock = true;+ if (rect.bottom >= view.innerHeight - 10 && rect.top > view.innerHeight * 0.6) targetInBottomDock = true;+ }+ if (!targetInTopDock) top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));if (bodyStyle) {- top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));- bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));+ if (!targetInTopDock) top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));}const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');const vw = view.innerWidth || 800;const vh = view.innerHeight || 600;for (const c of candidates) {+ if (target && _isComposedAncestor(c, target)) continue;if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;const cs = view.getComputedStyle(c);if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;@@ -617,7 +628,7 @@if (!el?.isConnected) return;try {const view = el.ownerDocument?.defaultView || window;- const insets = _getViewportDockedInsets(view);+ const insets = _getViewportDockedInsets(view, el);// A viewport-only rect check misses elements clipped by a scrollable// ancestor even when their bounding box is inside the viewport.if (!_isFullyVisibleForInteraction(el, insets)) {@@ -639,7 +650,7 @@if (!el?.isConnected) return false;const view = el.ownerDocument?.defaultView || window;const rect = el.getBoundingClientRect();- const insets = dockedInsets || _getViewportDockedInsets(view);+ const insets = dockedInsets || _getViewportDockedInsets(view, el);if (rect.width < 1|| rect.height < 1@@ -2280,7 +2291,7 @@if (topmost && !_hitTestMatchesTarget(el, topmost)) {const bRect = topmost.getBoundingClientRect();if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });r = el.getBoundingClientRect();cx = Math.round(r.left + r.width / 2);cy = Math.round(r.top + r.height / 2);diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjsindex adbf3b6ab..1501f4321 100644--- a/test/fixtures/run.mjs+++ b/test/fixtures/run.mjs@@ -2290,6 +2290,102 @@ test('Chrome Agent: modal auto-select ignores background/hidden clickables and k});// βββ occlusion ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+async function assertDockedControlsKeepScroll(page, browserKind) {+ for (const position of ['fixed', 'sticky']) {+ await setupContentHtml(page, `<!doctype html>+ <style>+ html { scroll-padding-top: 80px; scroll-padding-bottom: 60px; }+ body { margin: 0; height: 1700px; }+ header { position: ${position}; top: 0; height: 80px; width: 100%; background: white; }+ .toolbar { position: fixed; bottom: 0; height: 60px; width: 100%; background: white; }+ </style>+ <header><button id="top-control" onclick="window.__topClicked = true">Top control</button><input id="top-input" aria-label="Top input"></header>+ <div class="toolbar"><button id="bottom-control" onclick="window.__bottomClicked = true">Bottom control</button><input id="bottom-check" type="checkbox" aria-label="Bottom check"></div>+ `, browserKind);+ await page.evaluate(() => window.scrollTo(0, 300));+ const before = await page.evaluate(() => window.scrollY);+ for (const [selector, flag] of [['#top-control', '__topClicked'], ['#bottom-control', '__bottomClicked']]) {+ const result = await call(page, 'click', { selector });+ const after = await page.evaluate(name => ({ scrollY: window.scrollY, clicked: window[name] === true }), flag);+ if (!result?.success || !after.clicked || Math.abs(after.scrollY - before) > 1) {+ throw new Error(`${browserKind} ${position} ${selector} jumped from ${before}: ${JSON.stringify({ result, after })}`);+ }+ }+ const inputRef = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('top-input')));+ const typed = await call(page, 'type_ax', { ref_id: inputRef, text: 'Ada', clear: true });+ const typedState = await page.evaluate(() => ({+ scrollY: window.scrollY,+ value: document.getElementById('top-input').value,+ }));+ if (!typed?.success || typedState.value !== 'Ada' || Math.abs(typedState.scrollY - before) > 1) {+ throw new Error(`${browserKind} ${position} input jumped from ${before}: ${JSON.stringify({ typed, typedState })}`);+ }+ const checkRef = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('bottom-check')));+ const checked = await call(page, 'set_checked', { ref_id: checkRef, checked: true });+ const checkedState = await page.evaluate(() => ({+ scrollY: window.scrollY,+ checked: document.getElementById('bottom-check').checked,+ }));+ if (!checked?.success || !checkedState.checked || Math.abs(checkedState.scrollY - before) > 1) {+ throw new Error(`${browserKind} ${position} checkbox jumped from ${before}: ${JSON.stringify({ checked, checkedState })}`);+ }+ if (browserKind === 'chrome') {+ const rect = await call(page, 'ax_resolve_rect', { ref_id: inputRef });+ const afterRect = await page.evaluate(() => window.scrollY);+ if (!rect?.success || Math.abs(afterRect - before) > 1) {+ throw new Error(`Chrome ${position} rect resolution jumped from ${before}: ${JSON.stringify({ rect, afterRect })}`);+ }+ }+ }+}++test('Chrome: controls inside docked bars keep the document scroll position', page =>+ assertDockedControlsKeepScroll(page, 'chrome'));+firefoxTest('Firefox: controls inside docked bars keep the document scroll position', page =>+ assertDockedControlsKeepScroll(page, 'firefox'));++async function assertTopObstructionScrollDirection(page, browserKind) {+ await setupContentHtml(page, `<!doctype html>+ <style>+ body { margin: 0; height: 1700px; }+ #obstruction { position: fixed; top: 0; left: 0; width: 100%; height: 80px; z-index: 20; background: white; }+ #target { position: absolute; top: 320px; left: 60px; width: 120px; height: 100px; }+ #row { position: absolute; top: 320px; left: 220px; width: 120px; height: 40px; }+ </style>+ <button id="target" onclick="window.__targetClicked = true">Partly covered</button>+ <div id="row" role="listitem">Covered row</div>+ <div id="obstruction">Floating obstruction</div>+ `, browserKind);+ await page.evaluate(() => window.scrollTo(0, 300));+ const click = await call(page, 'click', { selector: '#target' });+ const afterClick = await page.evaluate(() => ({+ scrollY: window.scrollY,+ clicked: window.__targetClicked === true,+ top: document.getElementById('target').getBoundingClientRect().top,+ }));+ if (!click?.success || !afterClick.clicked || afterClick.scrollY >= 300 || afterClick.top < 80) {+ throw new Error(`${browserKind} top obstruction retry scrolled the wrong way: ${JSON.stringify({ click, afterClick })}`);+ }+ if (browserKind === 'chrome') {+ await page.evaluate(() => window.scrollTo(0, 300));+ const refId = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('row')));+ const resolved = await call(page, 'ax_resolve_rect', { ref_id: refId, forClickFallback: true });+ const afterResolve = await page.evaluate(() => ({+ scrollY: window.scrollY,+ top: document.getElementById('row').getBoundingClientRect().top,+ }));+ if (!resolved?.success || !resolved.hitOk || afterResolve.scrollY >= 300 || afterResolve.top < 80+ || Math.abs(resolved.rect?.y - afterResolve.top) > 1) {+ throw new Error(`Chrome rect resolver scrolled the wrong way: ${JSON.stringify({ resolved, afterResolve })}`);+ }+ }+}++test('Chrome: top obstruction retry moves targets below the blocker', page =>+ assertTopObstructionScrollDirection(page, 'chrome'));+firefoxTest('Firefox: top obstruction retry moves targets below the blocker', page =>+ assertTopObstructionScrollDirection(page, 'firefox'));+async function assertInnerScrollerClearance(page, browserKind) {await setupContentHtml(page, `<!doctype html><style>