Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix page monitor review findings

Barack Sokullu9ae0433dc8cb
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 01c5fdd19..18cf7e9fe 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -122,13 +122,54 @@
};
const related = (a, b) => !!a && !!b && (a === b
|| (a !== document.body && a !== document.documentElement && a.contains?.(b)));
- function layoutAgentUserAt() {
- if (agentTurn && lastUserAt <= agentTurn.userAt) return agentTurn.userAt;
- for (const op of operations.values()) {
- if (op.dispatched && Date.now() <= op.until && Number.isFinite(op.userAt) && lastUserAt <= op.userAt) return op.userAt;
+ function layoutAgentOperation(op, el, shift) {
+ if (!op?.dispatched || Date.now() > Math.max(op.until, op.layoutUntil || 0)
+ || !Number.isFinite(op.userAt) || lastUserAt > op.userAt) return false;
+ const target = op.target || (op.focused ? op.focusTarget : null);
+ if (!target) return false;
+ // A layout shift is attributable only when its source is the operation's
+ // target, part of that target, or an ancestor that moves the target.
+ if (related(target, el) || related(el, target)) return true;
+ // Editing a target can move a following sibling. Associate only the
+ // matching displacement from a recent, expected input on that target.
+ for (const cause of op.layoutCauses || []) {
+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;
+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
+ if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)
+ || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;
+ return true;
+ }
+ // Scroll operations may change layout inside their actual scroll box.
+ // Ignore document-wide ancestors so an unrelated page shift is not claimed.
+ for (const region of op.scrollAncestors || []) {
+ if (!region?.isConnected || region === document.body || region === document.documentElement) continue;
+ const style = getComputedStyle(region);
+ const scrollableX = /^(auto|scroll|overlay)$/.test(style.overflowX) && region.scrollWidth > region.clientWidth + 1;
+ const scrollableY = /^(auto|scroll|overlay)$/.test(style.overflowY) && region.scrollHeight > region.clientHeight + 1;
+ if ((scrollableX || scrollableY) && related(region, el)) return true;
+ }
+ return false;
+ }
+ function layoutAgentUserAt(el, shift) {
+ if (!(el instanceof Element)) return undefined;
+ if (agentTurn && lastUserAt <= agentTurn.userAt && localOperation?.operationId) {
+ const op = operations.get(localOperation.operationId);
+ if (layoutAgentOperation(op, el, shift)) return op.userAt;
}
+ for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;
return undefined;
}
+ function rememberAgentLayout(op) {
+ if (!op?.target) return;
+ const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);
+ if (before && after && Math.abs(after.height - before.height) > 2) {
+ const causes = op.layoutCauses ||= [];
+ causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });
+ if (causes.length > 8) causes.splice(0, causes.length - 8);
+ }
+ if (after) op.layoutRect = after;
+ op.layoutUntil = Date.now() + 3000;
+ }
const rectFor = el => {
if (!(el instanceof Element) || !el.isConnected) return null;
const rect = el.getBoundingClientRect();
@@ -148,7 +189,7 @@
if (!hitChanged && !rectChanged) return false;
op.layoutInvalidated = true;
revision++; publishRevision();
- const source = layoutAgentUserAt() !== undefined ? 'agent' : 'page';
+ const source = layoutAgentUserAt(target || currentHit) !== undefined ? 'agent' : 'page';
send({ kind: 'dom', source, target: targetName(target || currentHit) });
return true;
}
@@ -244,16 +285,23 @@
}
function sampleDescendants(root, limit = 100) {
const nodes = [];
- let node = root?.firstElementChild || null;
- while (node) {
+ const stack = [];
+ const pushChildren = parent => {
+ const length = parent?.children?.length || 0;
+ if (length) stack.push({ parent, index: 0, length });
+ };
+ pushChildren(root);
+ pushChildren(root?.shadowRoot);
+ while (stack.length) {
+ const frame = stack[stack.length - 1];
+ if (frame.index >= frame.length) { stack.pop(); continue; }
+ const node = frame.parent.children[frame.index++];
if (nodes.length === limit) return { nodes, exceeded: true };
nodes.push(node);
- if (node.firstElementChild) {
- node = node.firstElementChild;
- continue;
- }
- while (node && node !== root && !node.nextElementSibling) node = node.parentElement;
- node = node && node !== root ? node.nextElementSibling : null;
+ pushChildren(node);
+ // Inherited host styles and CSS variables can alter visible descendants
+ // inside an open shadow root without changing the host's own signature.
+ pushChildren(node.shadowRoot);
}
return { nodes, exceeded: false };
}
@@ -319,7 +367,11 @@
const noteChange = (el, agentUserAt) => {
// Attribute the marked input/write's synchronous handlers to that dispatch.
// Later DOM writes stay observable, including writes on the same target.
- if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) return;
+ if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) {
+ const expectedAt = Number.isFinite(agentUserAt) ? agentUserAt : agentTurn?.userAt;
+ for (const op of operations.values()) if (op.userAt === expectedAt && related(op.target, el)) rememberAgentLayout(op);
+ return;
+ }
changed = true;
target ||= targetName(el);
if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';
@@ -405,8 +457,8 @@
if (!el || ignored(el) || !visible(el) || el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
const oldRect = source.previousRect, newRect = source.currentRect;
if (!oldRect || !newRect || !['x', 'y', 'width', 'height'].some(key => Math.abs(oldRect[key] - newRect[key]) > 2)) continue;
- records.push({ type: 'layout', target: el, layoutChanged: true,
- agentUserAt: layoutAgentUserAt() });
+ const agentUserAt = layoutAgentUserAt(el, { previousRect: source.previousRect, currentRect: source.currentRect, entry });
+ records.push({ type: 'layout', target: el, layoutChanged: true, agentUserAt });
if (records.length >= 32) break;
}
if (records.length >= 32) break;
@@ -460,6 +512,7 @@
const el = elementFor(event);
const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);
if (!op) return;
+ if (['input', 'beforeinput', 'change'].includes(event.type)) rememberAgentLayout(op);
const marker = { userAt: op.userAt };
agentTurn = marker;
if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {
@@ -629,6 +682,11 @@
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;
+ if (op.dispatched && op.target) {
+ op.targetRectAtDispatch = rectFor(op.target);
+ op.targetParentAtDispatch = op.target.parentElement;
+ op.layoutRect ||= op.targetRectAtDispatch;
+ }
operations.set(op.operationId, op);
}
function activatePreparedDispatch(params = {}) {
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 01c5fdd19..18cf7e9fe 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -122,13 +122,54 @@
};
const related = (a, b) => !!a && !!b && (a === b
|| (a !== document.body && a !== document.documentElement && a.contains?.(b)));
- function layoutAgentUserAt() {
- if (agentTurn && lastUserAt <= agentTurn.userAt) return agentTurn.userAt;
- for (const op of operations.values()) {
- if (op.dispatched && Date.now() <= op.until && Number.isFinite(op.userAt) && lastUserAt <= op.userAt) return op.userAt;
+ function layoutAgentOperation(op, el, shift) {
+ if (!op?.dispatched || Date.now() > Math.max(op.until, op.layoutUntil || 0)
+ || !Number.isFinite(op.userAt) || lastUserAt > op.userAt) return false;
+ const target = op.target || (op.focused ? op.focusTarget : null);
+ if (!target) return false;
+ // A layout shift is attributable only when its source is the operation's
+ // target, part of that target, or an ancestor that moves the target.
+ if (related(target, el) || related(el, target)) return true;
+ // Editing a target can move a following sibling. Associate only the
+ // matching displacement from a recent, expected input on that target.
+ for (const cause of op.layoutCauses || []) {
+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;
+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
+ if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)
+ || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;
+ return true;
+ }
+ // Scroll operations may change layout inside their actual scroll box.
+ // Ignore document-wide ancestors so an unrelated page shift is not claimed.
+ for (const region of op.scrollAncestors || []) {
+ if (!region?.isConnected || region === document.body || region === document.documentElement) continue;
+ const style = getComputedStyle(region);
+ const scrollableX = /^(auto|scroll|overlay)$/.test(style.overflowX) && region.scrollWidth > region.clientWidth + 1;
+ const scrollableY = /^(auto|scroll|overlay)$/.test(style.overflowY) && region.scrollHeight > region.clientHeight + 1;
+ if ((scrollableX || scrollableY) && related(region, el)) return true;
+ }
+ return false;
+ }
+ function layoutAgentUserAt(el, shift) {
+ if (!(el instanceof Element)) return undefined;
+ if (agentTurn && lastUserAt <= agentTurn.userAt && localOperation?.operationId) {
+ const op = operations.get(localOperation.operationId);
+ if (layoutAgentOperation(op, el, shift)) return op.userAt;
}
+ for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;
return undefined;
}
+ function rememberAgentLayout(op) {
+ if (!op?.target) return;
+ const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);
+ if (before && after && Math.abs(after.height - before.height) > 2) {
+ const causes = op.layoutCauses ||= [];
+ causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });
+ if (causes.length > 8) causes.splice(0, causes.length - 8);
+ }
+ if (after) op.layoutRect = after;
+ op.layoutUntil = Date.now() + 3000;
+ }
const rectFor = el => {
if (!(el instanceof Element) || !el.isConnected) return null;
const rect = el.getBoundingClientRect();
@@ -148,7 +189,7 @@
if (!hitChanged && !rectChanged) return false;
op.layoutInvalidated = true;
revision++; publishRevision();
- const source = layoutAgentUserAt() !== undefined ? 'agent' : 'page';
+ const source = layoutAgentUserAt(target || currentHit) !== undefined ? 'agent' : 'page';
send({ kind: 'dom', source, target: targetName(target || currentHit) });
return true;
}
@@ -244,16 +285,23 @@
}
function sampleDescendants(root, limit = 100) {
const nodes = [];
- let node = root?.firstElementChild || null;
- while (node) {
+ const stack = [];
+ const pushChildren = parent => {
+ const length = parent?.children?.length || 0;
+ if (length) stack.push({ parent, index: 0, length });
+ };
+ pushChildren(root);
+ pushChildren(root?.shadowRoot);
+ while (stack.length) {
+ const frame = stack[stack.length - 1];
+ if (frame.index >= frame.length) { stack.pop(); continue; }
+ const node = frame.parent.children[frame.index++];
if (nodes.length === limit) return { nodes, exceeded: true };
nodes.push(node);
- if (node.firstElementChild) {
- node = node.firstElementChild;
- continue;
- }
- while (node && node !== root && !node.nextElementSibling) node = node.parentElement;
- node = node && node !== root ? node.nextElementSibling : null;
+ pushChildren(node);
+ // Inherited host styles and CSS variables can alter visible descendants
+ // inside an open shadow root without changing the host's own signature.
+ pushChildren(node.shadowRoot);
}
return { nodes, exceeded: false };
}
@@ -319,7 +367,11 @@
const noteChange = (el, agentUserAt) => {
// Attribute the marked input/write's synchronous handlers to that dispatch.
// Later DOM writes stay observable, including writes on the same target.
- if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) return;
+ if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) {
+ const expectedAt = Number.isFinite(agentUserAt) ? agentUserAt : agentTurn?.userAt;
+ for (const op of operations.values()) if (op.userAt === expectedAt && related(op.target, el)) rememberAgentLayout(op);
+ return;
+ }
changed = true;
target ||= targetName(el);
if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';
@@ -405,8 +457,8 @@
if (!el || ignored(el) || !visible(el) || el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
const oldRect = source.previousRect, newRect = source.currentRect;
if (!oldRect || !newRect || !['x', 'y', 'width', 'height'].some(key => Math.abs(oldRect[key] - newRect[key]) > 2)) continue;
- records.push({ type: 'layout', target: el, layoutChanged: true,
- agentUserAt: layoutAgentUserAt() });
+ const agentUserAt = layoutAgentUserAt(el, { previousRect: source.previousRect, currentRect: source.currentRect, entry });
+ records.push({ type: 'layout', target: el, layoutChanged: true, agentUserAt });
if (records.length >= 32) break;
}
if (records.length >= 32) break;
@@ -460,6 +512,7 @@
const el = elementFor(event);
const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);
if (!op) return;
+ if (['input', 'beforeinput', 'change'].includes(event.type)) rememberAgentLayout(op);
const marker = { userAt: op.userAt };
agentTurn = marker;
if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {
@@ -629,6 +682,11 @@
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;
+ if (op.dispatched && op.target) {
+ op.targetRectAtDispatch = rectFor(op.target);
+ op.targetParentAtDispatch = op.target.parentElement;
+ op.layoutRect ||= op.targetRectAtDispatch;
+ }
operations.set(op.operationId, op);
}
function activatePreparedDispatch(params = {}) {
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index dc25d35bf..3d02cd870 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -113,6 +113,26 @@ test('Chrome AX preparation attributes actual scrolling without claiming field i
} finally { await browser.close(); }
});
+test('Chrome: unrelated layout shifts during an agent operation stay observable', async () => {
+ const { browser, page } = await fixture(chromium, 'chrome');
+ try {
+ await page.evaluate(() => {
+ const spacer = document.createElement('div'); spacer.id = 'unrelated-layout-shift';
+ spacer.style.height = '1px'; document.getElementById('agent').after(spacer);
+ });
+ await page.waitForTimeout(200);
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'active-layout-op', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'active-layout-op', kind: 'click', selector: '#agent' });
+ document.styleSheets[0].insertRule('#unrelated-layout-shift { height: 120px !important; }', document.styleSheets[0].cssRules.length);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.ok(await page.evaluate(() => feedback.some(event => event.kind === 'dom' && event.source !== 'agent')),
+ 'A shift in a sibling region must not be discarded as an expected agent effect');
+ } finally { await browser.close(); }
+});
+
async function installContentEditableFallback(page, build) {
const insertion = read(build, 'content/content.js').match(/^ async function _insertContentEditableText\([\s\S]*?^ }/m)?.[0];
assert.ok(insertion, 'Exercise the real contenteditable fallback with the real monitor');
@@ -1359,6 +1379,32 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: inherited host styles track visible changes inside open shadow roots`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const host = document.createElement('div'); host.id = 'shadow-inherited-style-host';
+ host.className = 'closed'; host.style.cssText = 'width:200px;height:80px';
+ host.attachShadow({ mode: 'open' }).innerHTML = '<style>:host { --result-display:none } :host(.open) { --result-display:block } #shadow-inherited-result { display:var(--result-display) }</style><div id="shadow-inherited-result">Shadow result</div>';
+ document.body.prepend(host);
+ });
+ await page.waitForTimeout(200);
+ await page.evaluate(() => { feedback = []; });
+ for (const shown of [true, false]) {
+ await page.evaluate(shown => {
+ feedback = [];
+ document.getElementById('shadow-inherited-style-host').classList.toggle('open', shown);
+ }, shown);
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.equal(await page.locator('#shadow-inherited-style-host').evaluate(host =>
+ getComputedStyle(host.shadowRoot.getElementById('shadow-inherited-result')).display !== 'none'), shown);
+ assert.ok(await page.evaluate(() => feedback.some(event => event.kind === 'dom'
+ && event.target === 'div#shadow-inherited-result')),
+ `A shadow descendant whose inherited visibility changed must be the feedback target: ${JSON.stringify(await page.evaluate(() => feedback))}`);
+ }
+ } finally { await browser.close(); }
+ });
+
test(`${build}: large ancestor visibility changes invalidate actions beyond the sampling budget`, async () => {
const { browser, page } = await fixture(engine, build);
try {