Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: preserve docked controls and correct obstruction scroll

Barack Sokulluc06464e3a941
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 21f14df47..abaef38ee 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -312,7 +312,7 @@
: null;
}
- function _getViewportDockedInsets(view = window) {
+ function _getViewportDockedInsets(view = window, target = null) {
let top = 0;
let bottom = 0;
try {
@@ -324,16 +324,27 @@
const n = parseFloat(val);
return Number.isFinite(n) && n > 0 ? n : 0;
};
- top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
- bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
+ let targetInTopDock = false;
+ let targetInBottomDock = false;
+ for (let node = target; node && node !== doc; node = _composedParent(node)) {
+ if (node.nodeType !== Node.ELEMENT_NODE) continue;
+ const style = view.getComputedStyle(node);
+ if (style.position !== 'fixed' && style.position !== 'sticky') continue;
+ const rect = node.getBoundingClientRect();
+ if (rect.top <= 10 && rect.bottom > 0 && rect.bottom < view.innerHeight * 0.4) targetInTopDock = true;
+ if (rect.bottom >= view.innerHeight - 10 && rect.top > view.innerHeight * 0.6) targetInBottomDock = true;
+ }
+ if (!targetInTopDock) top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
if (bodyStyle) {
- top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
- bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
+ if (!targetInTopDock) top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
}
const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');
const vw = view.innerWidth || 800;
const vh = view.innerHeight || 600;
for (const c of candidates) {
+ if (target && _isComposedAncestor(c, target)) continue;
if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;
const cs = view.getComputedStyle(c);
if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
@@ -358,7 +369,7 @@
if (!el?.isConnected) return;
try {
const view = el.ownerDocument?.defaultView || window;
- const insets = _getViewportDockedInsets(view);
+ const insets = _getViewportDockedInsets(view, el);
// A viewport-only rect check misses elements clipped by a scrollable
// ancestor even when their bounding box is inside the viewport.
if (!_isFullyVisibleForInteraction(el, insets)) {
@@ -380,7 +391,7 @@
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
- const insets = dockedInsets || _getViewportDockedInsets(view);
+ const insets = dockedInsets || _getViewportDockedInsets(view, el);
if (
rect.width < 1
|| rect.height < 1
@@ -1981,7 +1992,7 @@
if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {
const bRect = topmost.getBoundingClientRect();
if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {
- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });
r = el.getBoundingClientRect();
cx = Math.round(r.left + r.width / 2);
cy = Math.round(r.top + r.height / 2);
@@ -7956,7 +7967,7 @@
}
try { _scrollElementIntoClearView(el); } catch {}
showAgentWorkingTarget(el, 'ax_resolve_rect');
- const r = el.getBoundingClientRect();
+ let r = el.getBoundingClientRect();
const cx = r.left + r.width / 2;
const cy = r.top + r.height / 2;
const vw = window.innerWidth, vh = window.innerHeight;
@@ -8008,10 +8019,10 @@
try {
const bRect = topmost.getBoundingClientRect();
if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < vh * 0.45) {
- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
- const rNew = el.getBoundingClientRect();
- effectiveCx = rNew.left + rNew.width / 2;
- effectiveCy = rNew.top + rNew.height / 2;
+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });
+ r = el.getBoundingClientRect();
+ effectiveCx = r.left + r.width / 2;
+ effectiveCy = r.top + r.height / 2;
topmost = document.elementFromPoint(effectiveCx, effectiveCy);
if (topmost && (topmost === el || el.contains?.(topmost) || topmost.contains?.(el))) {
hitOk = true;
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index ed00a3446..7f1374d22 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -571,7 +571,7 @@
: null;
}
- function _getViewportDockedInsets(view = window) {
+ function _getViewportDockedInsets(view = window, target = null) {
let top = 0;
let bottom = 0;
try {
@@ -583,16 +583,27 @@
const n = parseFloat(val);
return Number.isFinite(n) && n > 0 ? n : 0;
};
- top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
- bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
+ let targetInTopDock = false;
+ let targetInBottomDock = false;
+ for (let node = target; node && node !== doc; node = _composedParent(node)) {
+ if (node.nodeType !== Node.ELEMENT_NODE) continue;
+ const style = view.getComputedStyle(node);
+ if (style.position !== 'fixed' && style.position !== 'sticky') continue;
+ const rect = node.getBoundingClientRect();
+ if (rect.top <= 10 && rect.bottom > 0 && rect.bottom < view.innerHeight * 0.4) targetInTopDock = true;
+ if (rect.bottom >= view.innerHeight - 10 && rect.top > view.innerHeight * 0.6) targetInBottomDock = true;
+ }
+ if (!targetInTopDock) top = Math.max(top, parsePadding(htmlStyle.scrollPaddingTop));
+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(htmlStyle.scrollPaddingBottom));
if (bodyStyle) {
- top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
- bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
+ if (!targetInTopDock) top = Math.max(top, parsePadding(bodyStyle.scrollPaddingTop));
+ if (!targetInBottomDock) bottom = Math.max(bottom, parsePadding(bodyStyle.scrollPaddingBottom));
}
const candidates = doc.querySelectorAll('header, nav, [role="banner"], [role="navigation"], [class*="header" i], [class*="navbar" i], [class*="toolbar" i]');
const vw = view.innerWidth || 800;
const vh = view.innerHeight || 600;
for (const c of candidates) {
+ if (target && _isComposedAncestor(c, target)) continue;
if (!c.isConnected || c.offsetWidth <= 0 || c.offsetHeight <= 0) continue;
const cs = view.getComputedStyle(c);
if (cs.position !== 'fixed' && cs.position !== 'sticky') continue;
@@ -617,7 +628,7 @@
if (!el?.isConnected) return;
try {
const view = el.ownerDocument?.defaultView || window;
- const insets = _getViewportDockedInsets(view);
+ const insets = _getViewportDockedInsets(view, el);
// A viewport-only rect check misses elements clipped by a scrollable
// ancestor even when their bounding box is inside the viewport.
if (!_isFullyVisibleForInteraction(el, insets)) {
@@ -639,7 +650,7 @@
if (!el?.isConnected) return false;
const view = el.ownerDocument?.defaultView || window;
const rect = el.getBoundingClientRect();
- const insets = dockedInsets || _getViewportDockedInsets(view);
+ const insets = dockedInsets || _getViewportDockedInsets(view, el);
if (
rect.width < 1
|| rect.height < 1
@@ -2280,7 +2291,7 @@
if (topmost && !_hitTestMatchesTarget(el, topmost)) {
const bRect = topmost.getBoundingClientRect();
if (bRect.top <= 10 && bRect.bottom > r.top && bRect.bottom < window.innerHeight * 0.45) {
- window.scrollBy({ top: bRect.bottom - r.top + 20, behavior: 'instant' });
+ window.scrollBy({ top: r.top - bRect.bottom - 20, behavior: 'instant' });
r = el.getBoundingClientRect();
cx = Math.round(r.left + r.width / 2);
cy = Math.round(r.top + r.height / 2);
diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs
index adbf3b6ab..1501f4321 100644
--- a/test/fixtures/run.mjs
+++ b/test/fixtures/run.mjs
@@ -2290,6 +2290,102 @@ test('Chrome Agent: modal auto-select ignores background/hidden clickables and k
});
// ─── occlusion ────────────────────────────────────────────────────────────
+async function assertDockedControlsKeepScroll(page, browserKind) {
+ for (const position of ['fixed', 'sticky']) {
+ await setupContentHtml(page, `<!doctype html>
+ <style>
+ html { scroll-padding-top: 80px; scroll-padding-bottom: 60px; }
+ body { margin: 0; height: 1700px; }
+ header { position: ${position}; top: 0; height: 80px; width: 100%; background: white; }
+ .toolbar { position: fixed; bottom: 0; height: 60px; width: 100%; background: white; }
+ </style>
+ <header><button id="top-control" onclick="window.__topClicked = true">Top control</button><input id="top-input" aria-label="Top input"></header>
+ <div class="toolbar"><button id="bottom-control" onclick="window.__bottomClicked = true">Bottom control</button><input id="bottom-check" type="checkbox" aria-label="Bottom check"></div>
+ `, browserKind);
+ await page.evaluate(() => window.scrollTo(0, 300));
+ const before = await page.evaluate(() => window.scrollY);
+ for (const [selector, flag] of [['#top-control', '__topClicked'], ['#bottom-control', '__bottomClicked']]) {
+ const result = await call(page, 'click', { selector });
+ const after = await page.evaluate(name => ({ scrollY: window.scrollY, clicked: window[name] === true }), flag);
+ if (!result?.success || !after.clicked || Math.abs(after.scrollY - before) > 1) {
+ throw new Error(`${browserKind} ${position} ${selector} jumped from ${before}: ${JSON.stringify({ result, after })}`);
+ }
+ }
+ const inputRef = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('top-input')));
+ const typed = await call(page, 'type_ax', { ref_id: inputRef, text: 'Ada', clear: true });
+ const typedState = await page.evaluate(() => ({
+ scrollY: window.scrollY,
+ value: document.getElementById('top-input').value,
+ }));
+ if (!typed?.success || typedState.value !== 'Ada' || Math.abs(typedState.scrollY - before) > 1) {
+ throw new Error(`${browserKind} ${position} input jumped from ${before}: ${JSON.stringify({ typed, typedState })}`);
+ }
+ const checkRef = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('bottom-check')));
+ const checked = await call(page, 'set_checked', { ref_id: checkRef, checked: true });
+ const checkedState = await page.evaluate(() => ({
+ scrollY: window.scrollY,
+ checked: document.getElementById('bottom-check').checked,
+ }));
+ if (!checked?.success || !checkedState.checked || Math.abs(checkedState.scrollY - before) > 1) {
+ throw new Error(`${browserKind} ${position} checkbox jumped from ${before}: ${JSON.stringify({ checked, checkedState })}`);
+ }
+ if (browserKind === 'chrome') {
+ const rect = await call(page, 'ax_resolve_rect', { ref_id: inputRef });
+ const afterRect = await page.evaluate(() => window.scrollY);
+ if (!rect?.success || Math.abs(afterRect - before) > 1) {
+ throw new Error(`Chrome ${position} rect resolution jumped from ${before}: ${JSON.stringify({ rect, afterRect })}`);
+ }
+ }
+ }
+}
+
+test('Chrome: controls inside docked bars keep the document scroll position', page =>
+ assertDockedControlsKeepScroll(page, 'chrome'));
+firefoxTest('Firefox: controls inside docked bars keep the document scroll position', page =>
+ assertDockedControlsKeepScroll(page, 'firefox'));
+
+async function assertTopObstructionScrollDirection(page, browserKind) {
+ await setupContentHtml(page, `<!doctype html>
+ <style>
+ body { margin: 0; height: 1700px; }
+ #obstruction { position: fixed; top: 0; left: 0; width: 100%; height: 80px; z-index: 20; background: white; }
+ #target { position: absolute; top: 320px; left: 60px; width: 120px; height: 100px; }
+ #row { position: absolute; top: 320px; left: 220px; width: 120px; height: 40px; }
+ </style>
+ <button id="target" onclick="window.__targetClicked = true">Partly covered</button>
+ <div id="row" role="listitem">Covered row</div>
+ <div id="obstruction">Floating obstruction</div>
+ `, browserKind);
+ await page.evaluate(() => window.scrollTo(0, 300));
+ const click = await call(page, 'click', { selector: '#target' });
+ const afterClick = await page.evaluate(() => ({
+ scrollY: window.scrollY,
+ clicked: window.__targetClicked === true,
+ top: document.getElementById('target').getBoundingClientRect().top,
+ }));
+ if (!click?.success || !afterClick.clicked || afterClick.scrollY >= 300 || afterClick.top < 80) {
+ throw new Error(`${browserKind} top obstruction retry scrolled the wrong way: ${JSON.stringify({ click, afterClick })}`);
+ }
+ if (browserKind === 'chrome') {
+ await page.evaluate(() => window.scrollTo(0, 300));
+ const refId = await page.evaluate(() => window.__wb_ax_ref(document.getElementById('row')));
+ const resolved = await call(page, 'ax_resolve_rect', { ref_id: refId, forClickFallback: true });
+ const afterResolve = await page.evaluate(() => ({
+ scrollY: window.scrollY,
+ top: document.getElementById('row').getBoundingClientRect().top,
+ }));
+ if (!resolved?.success || !resolved.hitOk || afterResolve.scrollY >= 300 || afterResolve.top < 80
+ || Math.abs(resolved.rect?.y - afterResolve.top) > 1) {
+ throw new Error(`Chrome rect resolver scrolled the wrong way: ${JSON.stringify({ resolved, afterResolve })}`);
+ }
+ }
+}
+
+test('Chrome: top obstruction retry moves targets below the blocker', page =>
+ assertTopObstructionScrollDirection(page, 'chrome'));
+firefoxTest('Firefox: top obstruction retry moves targets below the blocker', page =>
+ assertTopObstructionScrollDirection(page, 'firefox'));
+
async function assertInnerScrollerClearance(page, browserKind) {
await setupContentHtml(page, `<!doctype html>
<style>