Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix page monitor review findings
Barack Sokullu9ae0433dc8cb
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 01c5fdd19..18cf7e9fe 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -122,13 +122,54 @@};const related = (a, b) => !!a && !!b && (a === b|| (a !== document.body && a !== document.documentElement && a.contains?.(b)));- function layoutAgentUserAt() {- if (agentTurn && lastUserAt <= agentTurn.userAt) return agentTurn.userAt;- for (const op of operations.values()) {- if (op.dispatched && Date.now() <= op.until && Number.isFinite(op.userAt) && lastUserAt <= op.userAt) return op.userAt;+ function layoutAgentOperation(op, el, shift) {+ if (!op?.dispatched || Date.now() > Math.max(op.until, op.layoutUntil || 0)+ || !Number.isFinite(op.userAt) || lastUserAt > op.userAt) return false;+ const target = op.target || (op.focused ? op.focusTarget : null);+ if (!target) return false;+ // A layout shift is attributable only when its source is the operation's+ // target, part of that target, or an ancestor that moves the target.+ if (related(target, el) || related(el, target)) return true;+ // Editing a target can move a following sibling. Associate only the+ // matching displacement from a recent, expected input on that target.+ for (const cause of op.layoutCauses || []) {+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;+ if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)+ || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;+ return true;+ }+ // Scroll operations may change layout inside their actual scroll box.+ // Ignore document-wide ancestors so an unrelated page shift is not claimed.+ for (const region of op.scrollAncestors || []) {+ if (!region?.isConnected || region === document.body || region === document.documentElement) continue;+ const style = getComputedStyle(region);+ const scrollableX = /^(auto|scroll|overlay)$/.test(style.overflowX) && region.scrollWidth > region.clientWidth + 1;+ const scrollableY = /^(auto|scroll|overlay)$/.test(style.overflowY) && region.scrollHeight > region.clientHeight + 1;+ if ((scrollableX || scrollableY) && related(region, el)) return true;+ }+ return false;+ }+ function layoutAgentUserAt(el, shift) {+ if (!(el instanceof Element)) return undefined;+ if (agentTurn && lastUserAt <= agentTurn.userAt && localOperation?.operationId) {+ const op = operations.get(localOperation.operationId);+ if (layoutAgentOperation(op, el, shift)) return op.userAt;}+ for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;return undefined;}+ function rememberAgentLayout(op) {+ if (!op?.target) return;+ const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);+ if (before && after && Math.abs(after.height - before.height) > 2) {+ const causes = op.layoutCauses ||= [];+ causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });+ if (causes.length > 8) causes.splice(0, causes.length - 8);+ }+ if (after) op.layoutRect = after;+ op.layoutUntil = Date.now() + 3000;+ }const rectFor = el => {if (!(el instanceof Element) || !el.isConnected) return null;const rect = el.getBoundingClientRect();@@ -148,7 +189,7 @@if (!hitChanged && !rectChanged) return false;op.layoutInvalidated = true;revision++; publishRevision();- const source = layoutAgentUserAt() !== undefined ? 'agent' : 'page';+ const source = layoutAgentUserAt(target || currentHit) !== undefined ? 'agent' : 'page';send({ kind: 'dom', source, target: targetName(target || currentHit) });return true;}@@ -244,16 +285,23 @@}function sampleDescendants(root, limit = 100) {const nodes = [];- let node = root?.firstElementChild || null;- while (node) {+ const stack = [];+ const pushChildren = parent => {+ const length = parent?.children?.length || 0;+ if (length) stack.push({ parent, index: 0, length });+ };+ pushChildren(root);+ pushChildren(root?.shadowRoot);+ while (stack.length) {+ const frame = stack[stack.length - 1];+ if (frame.index >= frame.length) { stack.pop(); continue; }+ const node = frame.parent.children[frame.index++];if (nodes.length === limit) return { nodes, exceeded: true };nodes.push(node);- if (node.firstElementChild) {- node = node.firstElementChild;- continue;- }- while (node && node !== root && !node.nextElementSibling) node = node.parentElement;- node = node && node !== root ? node.nextElementSibling : null;+ pushChildren(node);+ // Inherited host styles and CSS variables can alter visible descendants+ // inside an open shadow root without changing the host's own signature.+ pushChildren(node.shadowRoot);}return { nodes, exceeded: false };}@@ -319,7 +367,11 @@const noteChange = (el, agentUserAt) => {// Attribute the marked input/write's synchronous handlers to that dispatch.// Later DOM writes stay observable, including writes on the same target.- if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) return;+ if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) {+ const expectedAt = Number.isFinite(agentUserAt) ? agentUserAt : agentTurn?.userAt;+ for (const op of operations.values()) if (op.userAt === expectedAt && related(op.target, el)) rememberAgentLayout(op);+ return;+ }changed = true;target ||= targetName(el);if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';@@ -405,8 +457,8 @@if (!el || ignored(el) || !visible(el) || el.getAnimations?.().some(animation => animation.playState === 'running')) continue;const oldRect = source.previousRect, newRect = source.currentRect;if (!oldRect || !newRect || !['x', 'y', 'width', 'height'].some(key => Math.abs(oldRect[key] - newRect[key]) > 2)) continue;- records.push({ type: 'layout', target: el, layoutChanged: true,- agentUserAt: layoutAgentUserAt() });+ const agentUserAt = layoutAgentUserAt(el, { previousRect: source.previousRect, currentRect: source.currentRect, entry });+ records.push({ type: 'layout', target: el, layoutChanged: true, agentUserAt });if (records.length >= 32) break;}if (records.length >= 32) break;@@ -460,6 +512,7 @@const el = elementFor(event);const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);if (!op) return;+ if (['input', 'beforeinput', 'change'].includes(event.type)) rememberAgentLayout(op);const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {@@ -629,6 +682,11 @@op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;+ if (op.dispatched && op.target) {+ op.targetRectAtDispatch = rectFor(op.target);+ op.targetParentAtDispatch = op.target.parentElement;+ op.layoutRect ||= op.targetRectAtDispatch;+ }operations.set(op.operationId, op);}function activatePreparedDispatch(params = {}) {diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 01c5fdd19..18cf7e9fe 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -122,13 +122,54 @@};const related = (a, b) => !!a && !!b && (a === b|| (a !== document.body && a !== document.documentElement && a.contains?.(b)));- function layoutAgentUserAt() {- if (agentTurn && lastUserAt <= agentTurn.userAt) return agentTurn.userAt;- for (const op of operations.values()) {- if (op.dispatched && Date.now() <= op.until && Number.isFinite(op.userAt) && lastUserAt <= op.userAt) return op.userAt;+ function layoutAgentOperation(op, el, shift) {+ if (!op?.dispatched || Date.now() > Math.max(op.until, op.layoutUntil || 0)+ || !Number.isFinite(op.userAt) || lastUserAt > op.userAt) return false;+ const target = op.target || (op.focused ? op.focusTarget : null);+ if (!target) return false;+ // A layout shift is attributable only when its source is the operation's+ // target, part of that target, or an ancestor that moves the target.+ if (related(target, el) || related(el, target)) return true;+ // Editing a target can move a following sibling. Associate only the+ // matching displacement from a recent, expected input on that target.+ for (const cause of op.layoutCauses || []) {+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;+ if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)+ || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;+ return true;+ }+ // Scroll operations may change layout inside their actual scroll box.+ // Ignore document-wide ancestors so an unrelated page shift is not claimed.+ for (const region of op.scrollAncestors || []) {+ if (!region?.isConnected || region === document.body || region === document.documentElement) continue;+ const style = getComputedStyle(region);+ const scrollableX = /^(auto|scroll|overlay)$/.test(style.overflowX) && region.scrollWidth > region.clientWidth + 1;+ const scrollableY = /^(auto|scroll|overlay)$/.test(style.overflowY) && region.scrollHeight > region.clientHeight + 1;+ if ((scrollableX || scrollableY) && related(region, el)) return true;+ }+ return false;+ }+ function layoutAgentUserAt(el, shift) {+ if (!(el instanceof Element)) return undefined;+ if (agentTurn && lastUserAt <= agentTurn.userAt && localOperation?.operationId) {+ const op = operations.get(localOperation.operationId);+ if (layoutAgentOperation(op, el, shift)) return op.userAt;}+ for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;return undefined;}+ function rememberAgentLayout(op) {+ if (!op?.target) return;+ const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);+ if (before && after && Math.abs(after.height - before.height) > 2) {+ const causes = op.layoutCauses ||= [];+ causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });+ if (causes.length > 8) causes.splice(0, causes.length - 8);+ }+ if (after) op.layoutRect = after;+ op.layoutUntil = Date.now() + 3000;+ }const rectFor = el => {if (!(el instanceof Element) || !el.isConnected) return null;const rect = el.getBoundingClientRect();@@ -148,7 +189,7 @@if (!hitChanged && !rectChanged) return false;op.layoutInvalidated = true;revision++; publishRevision();- const source = layoutAgentUserAt() !== undefined ? 'agent' : 'page';+ const source = layoutAgentUserAt(target || currentHit) !== undefined ? 'agent' : 'page';send({ kind: 'dom', source, target: targetName(target || currentHit) });return true;}@@ -244,16 +285,23 @@}function sampleDescendants(root, limit = 100) {const nodes = [];- let node = root?.firstElementChild || null;- while (node) {+ const stack = [];+ const pushChildren = parent => {+ const length = parent?.children?.length || 0;+ if (length) stack.push({ parent, index: 0, length });+ };+ pushChildren(root);+ pushChildren(root?.shadowRoot);+ while (stack.length) {+ const frame = stack[stack.length - 1];+ if (frame.index >= frame.length) { stack.pop(); continue; }+ const node = frame.parent.children[frame.index++];if (nodes.length === limit) return { nodes, exceeded: true };nodes.push(node);- if (node.firstElementChild) {- node = node.firstElementChild;- continue;- }- while (node && node !== root && !node.nextElementSibling) node = node.parentElement;- node = node && node !== root ? node.nextElementSibling : null;+ pushChildren(node);+ // Inherited host styles and CSS variables can alter visible descendants+ // inside an open shadow root without changing the host's own signature.+ pushChildren(node.shadowRoot);}return { nodes, exceeded: false };}@@ -319,7 +367,11 @@const noteChange = (el, agentUserAt) => {// Attribute the marked input/write's synchronous handlers to that dispatch.// Later DOM writes stay observable, including writes on the same target.- if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) return;+ if ((Number.isFinite(agentUserAt) && lastUserAt <= agentUserAt) || (agentTurn && lastUserAt <= agentTurn.userAt)) {+ const expectedAt = Number.isFinite(agentUserAt) ? agentUserAt : agentTurn?.userAt;+ for (const op of operations.values()) if (op.userAt === expectedAt && related(op.target, el)) rememberAgentLayout(op);+ return;+ }changed = true;target ||= targetName(el);if (userTurn || (Date.now() - lastUserAt < 1500 && related(lastUserTarget, el))) source = 'user';@@ -405,8 +457,8 @@if (!el || ignored(el) || !visible(el) || el.getAnimations?.().some(animation => animation.playState === 'running')) continue;const oldRect = source.previousRect, newRect = source.currentRect;if (!oldRect || !newRect || !['x', 'y', 'width', 'height'].some(key => Math.abs(oldRect[key] - newRect[key]) > 2)) continue;- records.push({ type: 'layout', target: el, layoutChanged: true,- agentUserAt: layoutAgentUserAt() });+ const agentUserAt = layoutAgentUserAt(el, { previousRect: source.previousRect, currentRect: source.currentRect, entry });+ records.push({ type: 'layout', target: el, layoutChanged: true, agentUserAt });if (records.length >= 32) break;}if (records.length >= 32) break;@@ -460,6 +512,7 @@const el = elementFor(event);const op = expected(['input', 'beforeinput', 'change', 'keydown'].includes(event.type) ? 'input' : 'click', el, event);if (!op) return;+ if (['input', 'beforeinput', 'change'].includes(event.type)) rememberAgentLayout(op);const marker = { userAt: op.userAt };agentTurn = marker;if (op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) {@@ -629,6 +682,11 @@op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;+ if (op.dispatched && op.target) {+ op.targetRectAtDispatch = rectFor(op.target);+ op.targetParentAtDispatch = op.target.parentElement;+ op.layoutRect ||= op.targetRectAtDispatch;+ }operations.set(op.operationId, op);}function activatePreparedDispatch(params = {}) {diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex dc25d35bf..3d02cd870 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -113,6 +113,26 @@ test('Chrome AX preparation attributes actual scrolling without claiming field i} finally { await browser.close(); }});+test('Chrome: unrelated layout shifts during an agent operation stay observable', async () => {+ const { browser, page } = await fixture(chromium, 'chrome');+ try {+ await page.evaluate(() => {+ const spacer = document.createElement('div'); spacer.id = 'unrelated-layout-shift';+ spacer.style.height = '1px'; document.getElementById('agent').after(spacer);+ });+ await page.waitForTimeout(200);+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'active-layout-op', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'active-layout-op', kind: 'click', selector: '#agent' });+ document.styleSheets[0].insertRule('#unrelated-layout-shift { height: 120px !important; }', document.styleSheets[0].cssRules.length);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.ok(await page.evaluate(() => feedback.some(event => event.kind === 'dom' && event.source !== 'agent')),+ 'A shift in a sibling region must not be discarded as an expected agent effect');+ } finally { await browser.close(); }+});+async function installContentEditableFallback(page, build) {const insertion = read(build, 'content/content.js').match(/^ async function _insertContentEditableText\([\s\S]*?^ }/m)?.[0];assert.ok(insertion, 'Exercise the real contenteditable fallback with the real monitor');@@ -1359,6 +1379,32 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: inherited host styles track visible changes inside open shadow roots`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const host = document.createElement('div'); host.id = 'shadow-inherited-style-host';+ host.className = 'closed'; host.style.cssText = 'width:200px;height:80px';+ host.attachShadow({ mode: 'open' }).innerHTML = '<style>:host { --result-display:none } :host(.open) { --result-display:block } #shadow-inherited-result { display:var(--result-display) }</style><div id="shadow-inherited-result">Shadow result</div>';+ document.body.prepend(host);+ });+ await page.waitForTimeout(200);+ await page.evaluate(() => { feedback = []; });+ for (const shown of [true, false]) {+ await page.evaluate(shown => {+ feedback = [];+ document.getElementById('shadow-inherited-style-host').classList.toggle('open', shown);+ }, shown);+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.equal(await page.locator('#shadow-inherited-style-host').evaluate(host =>+ getComputedStyle(host.shadowRoot.getElementById('shadow-inherited-result')).display !== 'none'), shown);+ assert.ok(await page.evaluate(() => feedback.some(event => event.kind === 'dom'+ && event.target === 'div#shadow-inherited-result')),+ `A shadow descendant whose inherited visibility changed must be the feedback target: ${JSON.stringify(await page.evaluate(() => feedback))}`);+ }+ } finally { await browser.close(); }+ });+test(`${build}: large ancestor visibility changes invalidate actions beyond the sampling budget`, async () => {const { browser, page } = await fixture(engine, build);try {