Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix fenced iframe click monitoring
Barack Sokullu0fce52d2f2ee
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 3e35ada46..0f058daea 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -37717,7 +37717,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai}if (binding?.token && Number.isInteger(binding.frameId)) {throwIfEarlyCdpAborted();- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, frameId: binding.frameId });+ await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, frameId: binding.frameId, fenceOnly: true });dispatched = true;markEarlyCdpDispatched();const response = await chrome.tabs.sendMessage(tabId, {@@ -37782,12 +37782,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai}const selected = candidates[0];throwIfEarlyCdpAborted();- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, frameId: selected.frameId });+ const dispatchGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'click', selector, frameId: selected.frameId, fenceOnly: true, prepareMonitor: true,+ });dispatched = true;markEarlyCdpDispatched();const clicked = await chrome.scripting.executeScript({target: { tabId, frameIds: [selected.frameId] },- func: (sel, matchIndex) => {+ func: (sel, matchIndex, monitorGuard) => {let targetDispatched = false;try {const el = document.querySelectorAll(sel)[matchIndex];@@ -37797,6 +37799,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiconst cx = rect.left + rect.width / 2;const cy = rect.top + rect.height / 2;const opts = { bubbles: true, cancelable: true, view: window, clientX: cx, clientY: cy, button: 0 };+ if (monitorGuard?.operationId) {+ if (typeof window.__wbPageMonitor?.activatePreparedDispatch !== 'function') {+ return { ok: false, url: location.href, reason: 'page-monitor-unavailable' };+ }+ window.__wbPageMonitor.activatePreparedDispatch({+ operationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,+ });+ }targetDispatched = true;try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}el.dispatchEvent(new MouseEvent('mousedown', opts));@@ -37808,7 +37818,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaireturn { ok: false, url: location.href, dispatched: targetDispatched, error: e.message };}},- args: [selector, requestedMatchIndex],+ args: [selector, requestedMatchIndex, dispatchGuard],});const result = clicked?.[0]?.result;return result?.okdiff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 6fe711fbf..db7b32e06 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -40,7 +40,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {error.code = 'page_feedback_pending';throw error;}- if (!details.release && details.kind === 'click' && details.navigationCandidate !== false) owner.clearNavigation();+ if (!details.release && !details.fenceOnly && details.kind === 'click' && details.navigationCandidate !== false) owner.clearNavigation();if (details.documentToken) {const frameId = owner.frameForDocument(details.documentToken);if (frameId === undefined && !details.release) {@@ -50,10 +50,18 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {details = { ...details, ...(frameId === undefined ? {} : { frameId }) };}owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);+ const { prepareMonitor = false, ...dispatchDetails } = details;+ const operationId = owner.operationId || token();+ const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };let acknowledgement;+ let preparationAcknowledgement;try {+ if (!details.release && details.fenceOnly && prepareMonitor) {+ preparationAcknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_prepare',+ params: { ...monitorParams, tool: dispatchDetails.tool || dispatchDetails.kind } }, { frameId: Number(details.frameId) || 0 });+ }acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_dispatch',- params: { ...details, runToken: owner.runToken, operationId: owner.operationId || token() } }, { frameId: Number(details.frameId) || 0 });+ params: monitorParams }, { frameId: Number(details.frameId) || 0 });} catch { /* restricted pages can still use background tools */ }if (!details.release && dispatchOwners.get(tabId) !== owner) {const error = new Error('Browser run ended before dispatch'); error.name = 'AbortError'; throw error;@@ -64,6 +72,11 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {error.code = 'page_feedback_pending';throw error;}+ if (!details.release && details.fenceOnly && prepareMonitor && acknowledgement?.guard && preparationAcknowledgement?.ready !== true) {+ const error = new Error('The browser could not prepare the page monitor before dispatch. Re-observe the page before acting.');+ error.code = 'page_feedback_pending';+ throw error;+ }// Clicks are correlated by the matching page event, not selector preparation.if (!details.release && details.kind === 'navigate') owner.dispatched(details);return acknowledgement?.guard;diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex afa340e2e..eae77576e 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -240,6 +240,21 @@if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);}}+ function sampleDescendants(root, limit = 100) {+ const nodes = [];+ let node = root?.firstElementChild || null;+ while (node) {+ if (nodes.length === limit) return { nodes, exceeded: true };+ nodes.push(node);+ if (node.firstElementChild) {+ node = node.firstElementChild;+ continue;+ }+ while (node && node !== root && !node.nextElementSibling) node = node.parentElement;+ node = node && node !== root ? node.nextElementSibling : null;+ }+ return { nodes, exceeded: false };+ }function observeRoot(root, budget) {if (!root || roots.has(root) || !observer) return;roots.add(root);@@ -313,17 +328,17 @@if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;- const interactive = record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [];- const all = record.target.querySelectorAll?.('*') || [];+ const sample = sampleDescendants(record.target);// A bounded layout sample cannot prove that inherited visibility left a// large subtree unchanged. Invalidate conservatively on an actual state// change, before the measurement cap can omit a late affected descendant.- if (all.length > 100 && (record.type === 'popover' || record.oldValue !== record.target.getAttribute(record.attributeName))) noteChange(record.target, record.agentUserAt);- const descendants = new Set([- ...Array.prototype.slice.call(interactive, 0, 100),- ...Array.prototype.slice.call(all, 0, 100),- ]);- for (const el of descendants)+ // Stop walking at the cap instead of materializing the whole subtree.+ if (sample.exceeded) {+ if (record.type === 'popover' || record.oldValue !== record.target.getAttribute(record.attributeName))+ noteChange(record.target, record.agentUserAt);+ continue;+ }+ for (const el of sample.nodes)changes.push({ type: 'layout', target: el, agentUserAt: record.agentUserAt });}let measured = 0;@@ -608,6 +623,21 @@op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;operations.set(op.operationId, op);}+ function activatePreparedDispatch(params = {}) {+ const op = operations.get(params.operationId);+ const layoutChanged = coordinatePreparationShifted(op);+ if (!active || !op || layoutChanged || domTimer || unreported+ || lastUserAt > op.userAt+ || (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {+ const error = new Error('Browser changed during action preparation. Re-observe before acting.');+ error.code = 'page_feedback_pending';+ error.dispatched = false;+ throw error;+ }+ dispatch({ ...params, runToken });+ publishRevision();+ return { ready: true, operationId: op.operationId, revision };+ }const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js','ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);@@ -689,7 +719,7 @@}};api.runtime.onMessage.addListener(onMessage);- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, registerDecoration,+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,get active() { return active; },get disposed() { return disposed; },dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 088ec3f42..84299ba9b 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -34837,6 +34837,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d};}const selected = candidates[0];+ const dispatchGuard = await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {+ kind: 'click', selector, frameId: selected.frameId, fenceOnly: true, prepareMonitor: true,+ });+ throwIfContentPipelineAborted();const clickCode = `(() => {let targetDispatched = false;@@ -34846,6 +34850,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout del.scrollIntoView({ block: 'center', inline: 'center' });const rect = el.getBoundingClientRect();const opts = { bubbles: true, cancelable: true, view: window, clientX: rect.left + rect.width/2, clientY: rect.top + rect.height/2, button: 0 };+ const monitorGuard = ${JSON.stringify(dispatchGuard || null)};+ if (monitorGuard?.operationId) {+ if (typeof window.__wbPageMonitor?.activatePreparedDispatch !== 'function') {+ return { ok: false, url: location.href, reason: 'page-monitor-unavailable' };+ }+ window.__wbPageMonitor.activatePreparedDispatch({+ operationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,+ });+ }targetDispatched = true;try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}el.dispatchEvent(new MouseEvent('mousedown', opts));diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 6fe711fbf..db7b32e06 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -40,7 +40,7 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {error.code = 'page_feedback_pending';throw error;}- if (!details.release && details.kind === 'click' && details.navigationCandidate !== false) owner.clearNavigation();+ if (!details.release && !details.fenceOnly && details.kind === 'click' && details.navigationCandidate !== false) owner.clearNavigation();if (details.documentToken) {const frameId = owner.frameForDocument(details.documentToken);if (frameId === undefined && !details.release) {@@ -50,10 +50,18 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {details = { ...details, ...(frameId === undefined ? {} : { frameId }) };}owner.operationFrames.get(owner.operationId)?.add(Number(details.frameId) || 0);+ const { prepareMonitor = false, ...dispatchDetails } = details;+ const operationId = owner.operationId || token();+ const monitorParams = { ...dispatchDetails, runToken: owner.runToken, operationId };let acknowledgement;+ let preparationAcknowledgement;try {+ if (!details.release && details.fenceOnly && prepareMonitor) {+ preparationAcknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_prepare',+ params: { ...monitorParams, tool: dispatchDetails.tool || dispatchDetails.kind } }, { frameId: Number(details.frameId) || 0 });+ }acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_dispatch',- params: { ...details, runToken: owner.runToken, operationId: owner.operationId || token() } }, { frameId: Number(details.frameId) || 0 });+ params: monitorParams }, { frameId: Number(details.frameId) || 0 });} catch { /* restricted pages can still use background tools */ }if (!details.release && dispatchOwners.get(tabId) !== owner) {const error = new Error('Browser run ended before dispatch'); error.name = 'AbortError'; throw error;@@ -64,6 +72,11 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {error.code = 'page_feedback_pending';throw error;}+ if (!details.release && details.fenceOnly && prepareMonitor && acknowledgement?.guard && preparationAcknowledgement?.ready !== true) {+ const error = new Error('The browser could not prepare the page monitor before dispatch. Re-observe the page before acting.');+ error.code = 'page_feedback_pending';+ throw error;+ }// Clicks are correlated by the matching page event, not selector preparation.if (!details.release && details.kind === 'navigate') owner.dispatched(details);return acknowledgement?.guard;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex afa340e2e..eae77576e 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -240,6 +240,21 @@if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);}}+ function sampleDescendants(root, limit = 100) {+ const nodes = [];+ let node = root?.firstElementChild || null;+ while (node) {+ if (nodes.length === limit) return { nodes, exceeded: true };+ nodes.push(node);+ if (node.firstElementChild) {+ node = node.firstElementChild;+ continue;+ }+ while (node && node !== root && !node.nextElementSibling) node = node.parentElement;+ node = node && node !== root ? node.nextElementSibling : null;+ }+ return { nodes, exceeded: false };+ }function observeRoot(root, budget) {if (!root || roots.has(root) || !observer) return;roots.add(root);@@ -313,17 +328,17 @@if (record.type !== 'popover' && (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden', 'open', 'inert'].includes(record.attributeName))) continue;if (ignored(record.target)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;- const interactive = record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [];- const all = record.target.querySelectorAll?.('*') || [];+ const sample = sampleDescendants(record.target);// A bounded layout sample cannot prove that inherited visibility left a// large subtree unchanged. Invalidate conservatively on an actual state// change, before the measurement cap can omit a late affected descendant.- if (all.length > 100 && (record.type === 'popover' || record.oldValue !== record.target.getAttribute(record.attributeName))) noteChange(record.target, record.agentUserAt);- const descendants = new Set([- ...Array.prototype.slice.call(interactive, 0, 100),- ...Array.prototype.slice.call(all, 0, 100),- ]);- for (const el of descendants)+ // Stop walking at the cap instead of materializing the whole subtree.+ if (sample.exceeded) {+ if (record.type === 'popover' || record.oldValue !== record.target.getAttribute(record.attributeName))+ noteChange(record.target, record.agentUserAt);+ continue;+ }+ for (const el of sample.nodes)changes.push({ type: 'layout', target: el, agentUserAt: record.agentUserAt });}let measured = 0;@@ -608,6 +623,21 @@op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;operations.set(op.operationId, op);}+ function activatePreparedDispatch(params = {}) {+ const op = operations.get(params.operationId);+ const layoutChanged = coordinatePreparationShifted(op);+ if (!active || !op || layoutChanged || domTimer || unreported+ || lastUserAt > op.userAt+ || (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {+ const error = new Error('Browser changed during action preparation. Re-observe before acting.');+ error.code = 'page_feedback_pending';+ error.dispatched = false;+ throw error;+ }+ dispatch({ ...params, runToken });+ publishRevision();+ return { ready: true, operationId: op.operationId, revision };+ }const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js','ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);@@ -689,7 +719,7 @@}};api.runtime.onMessage.addListener(onMessage);- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, registerDecoration,+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,get active() { return active; },get disposed() { return disposed; },dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex de0905ec6..9eb6e9aaf 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -381,6 +381,37 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: frame click preparation installs a fence without claiming the click`, async () => {+ const agent = setup(Agent), tab = nextTab++, previousMessage = api.tabs.sendMessage;+ const messages = [];+ api.tabs.sendMessage = async (_tab, message, options) => {+ messages.push({ message, frameId: options?.frameId });+ if (message.action === 'page_monitor_dispatch') return { ready: true, guard: {+ documentToken: 'child-token', revision: 0, operationId: message.params.operationId,+ navigationCandidate: true,+ } };+ return { ready: true };+ };+ await agent._claimRunEntry(tab, 'interactive');+ try {+ bind(agent, tab, 2, 'child-doc', 'child-token');+ const run = agent._pageFeedbackRuns.get(tab);+ run.navigation = { at: Date.now(), url: '', kind: 'click', frameId: 2 };+ const guard = await beforePageAgentDispatch(api, tab, {+ kind: 'click', selector: '#inside', frameId: 2, fenceOnly: true, prepareMonitor: true,+ });+ assert.equal(guard.operationId, messages.find(entry => entry.message.action === 'page_monitor_dispatch').message.params.operationId);+ assert.deepEqual(messages.filter(entry => ['page_monitor_prepare', 'page_monitor_dispatch'].includes(entry.message.action))+ .map(entry => entry.message.action), ['page_monitor_prepare', 'page_monitor_dispatch']);+ assert.equal(messages.at(-1).frameId, 2);+ assert.equal(messages.at(-1).message.params.fenceOnly, true);+ assert.equal(run.navigation.kind, 'click', 'A preparation fence cannot consume an earlier navigation correlation');+ } finally {+ agent._releaseRunEntry(tab);+ api.tabs.sendMessage = previousMessage;+ }+ });+test(`${build}: intervention during the dispatch handshake blocks the prepared action`, async () => {const agent = setup(Agent), tab = nextTab++;await agent._claimRunEntry(tab, 'interactive');diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex da8ecb825..32e4fdb53 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -457,6 +457,75 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: fenced iframe clicks reject transport-gap input and attribute the dispatched click`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'iframe-click-stale', tool: 'click', selector: '#human' });+ deliver('page_monitor_dispatch', { operationId: 'iframe-click-stale', kind: 'click', selector: '#human', fenceOnly: true });+ });+ await page.locator('#human').click();+ await page.waitForFunction(() => feedback.some(event => event.kind === 'click' && event.source === 'user'), null, { timeout: 1000 });+ const blocked = await page.evaluate(() => {+ try {+ __wbPageMonitor.activatePreparedDispatch({ operationId: 'iframe-click-stale', kind: 'click',+ element: document.getElementById('human'), navigationCandidate: true });+ return null;+ } catch (error) { return { code: error.code, dispatched: error.dispatched }; }+ });+ assert.deepEqual(blocked, { code: 'page_feedback_pending', dispatched: false },+ 'The remote script must not claim a physical click received during the transport gap');++ await page.waitForTimeout(200);+ const accepted = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'iframe-click-live', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'iframe-click-live', kind: 'click', selector: '#agent', fenceOnly: true });+ try {+ __wbPageMonitor.activatePreparedDispatch({ operationId: 'iframe-click-live', kind: 'click',+ element: document.getElementById('agent'), navigationCandidate: true });+ document.getElementById('agent').click();+ return true;+ } catch { return false; }+ });+ assert.equal(accepted, true, 'An unchanged prepared target can activate immediately before the click');+ await page.waitForTimeout(200);+ assert.ok((await page.evaluate(() => feedback)).every(event => event.source === 'agent'+ && event.kind === 'activity' && event.operation === 'click'),+ 'The activated script click may correlate navigation but must not be reported as a user/page change');+ assert.equal(await page.locator('#status').textContent(), 'Agent changed this');+ } finally { await browser.close(); }+ });++ test(`${build}: large ancestor style changes stop descendant sampling at its cap`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ monitorEnabled = false; deliver('page_monitor_state');+ const large = document.createElement('section');+ large.innerHTML = '<span></span>'.repeat(250);+ document.body.append(large);+ monitorEnabled = true; deliver('page_monitor_state');+ });+ await page.waitForTimeout(100);+ await page.evaluate(() => {+ const original = Element.prototype.querySelectorAll;+ let count = 0;+ Element.prototype.querySelectorAll = function (selector) {+ if (this === document.body && selector === '*') count++;+ return original.call(this, selector);+ };+ window.wholeSubtreeQueryCount = () => count;+ feedback = [];+ document.body.classList.add('large-page-state-change');+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.source === 'page'), null, { timeout: 1000 });+ assert.equal(await page.evaluate(() => wholeSubtreeQueryCount()), 0,+ 'A common-ancestor mutation must not materialize its complete descendant list before applying the cap');+ } finally { await browser.close(); }+ });+test(`${build}: opacity-only visibility changes invalidate prepared actions`, async () => {const { browser, page } = await fixture(engine, build);try {