Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard CDP click fallbacks against page feedback
Barack Sokullu12b61a596c7d
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex efa20c080..0b00685fa 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -39890,8 +39890,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaireturn { success: true };},});- if (selResult?.deadlineExpired === true && selResult?.dispatched !== true) {+ if ((selResult?.noDispatch === true || selResult?.dispatched === false) && selResult?.outcomeUnknown !== true) {earlyCdpDispatchState.started = false;+ }+ if (selResult?.deadlineExpired === true && selResult?.dispatched !== true) {return {...selResult,success: false,diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex 4dc89cb79..eefe4074c 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -18,6 +18,16 @@ const SELECTOR_SCROLL_DISPATCH_SOURCE = `};`;+const PAGE_AGENT_DOM_ACTION_SOURCE = `+ const beforePageAgentDomAction = (guard, phase) => {+ if (!guard) return true;+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...guard, dispatchPhase: phase }), cancelable: true,+ });+ return window.dispatchEvent(event);+ };+`;+function throwIfPageFeedbackPending(result) {if (!result?.pageFeedbackPending) return;const error = new Error(pageFeedbackPendingResult().error);@@ -4444,6 +4454,17 @@ export class CDPClient {deadlineExpired: true,error: error || 'Click action deadline expired before click dispatch',};+ const pageFeedbackClickResult = (priorDispatchAttempted, focusDispatched = false) => {+ if (!priorDispatchAttempted && !focusDispatched) return pageFeedbackPendingResult();+ return {+ success: false,+ dispatched: true,+ outcomeUnknown: true,+ retryable: false,+ pageFeedbackPending: true,+ error: 'The page changed during click preparation. A prior click or focus may have taken effect; inspect the page before retrying.',+ };+ };if (info.inViewport && info.hitOk) {try {await this.armFileInputClickGuard(tabId);@@ -4534,21 +4555,31 @@ export class CDPClient {const priorDispatchAttempted = dispatchAttempted;const validation = await authorizeDispatch({ x: info.x, y: info.y, tag: info.tag });if (validation.success !== true) return validation;- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });dispatchAttempted = true;const clicked = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {objectId,- functionDeclaration: `function(actionDeadlineAt) {+ functionDeclaration: `function(actionDeadlineAt, pageGuard) {+ ${PAGE_AGENT_DOM_ACTION_SOURCE}if (actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt) return false;+ if (!this || !this.isConnected) return false;+ if (!beforePageAgentDomAction(pageGuard, 'focus'))+ return { pageFeedbackPending: true, noDispatch: true, dispatched: false };try { this.focus(); } catch {}if (actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt) return false;+ if (!this.isConnected || !beforePageAgentDomAction(pageGuard, 'click'))+ return { pageFeedbackPending: true, noDispatch: false, dispatched: true };this.click();return true;}`,- arguments: [{ value: deadlineAt }],+ arguments: [{ value: deadlineAt }, { value: pageGuard || null }],returnByValue: true,awaitPromise: false,});+ if (clicked?.result?.value?.pageFeedbackPending === true) {+ await this.consumeFileInputClickGuard(tabId);+ return pageFeedbackClickResult(priorDispatchAttempted, clicked.result.value.dispatched === true);+ }if (clicked?.result?.value === false) {return pageDeadlineResult(priorDispatchAttempted, 'Click action deadline expired before click dispatch');}@@ -4585,22 +4616,19 @@ export class CDPClient {}// Step 3: JS fallback for open shadow roots.- const selectorJSON = JSON.stringify(selector);throwIfAborted();await this.armFileInputClickGuard(tabId);throwIfAborted();const fallbackValidation = await authorizeDispatch({ x: info.x, y: info.y, tag: info.tag });if (fallbackValidation.success !== true) return fallbackValidation;const priorDispatchAttempted = dispatchAttempted;- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });dispatchAttempted = true;- const fb = await this.evaluate(tabId, `- (() => {- const actionDeadlineAt = ${deadlineAt};+ const fb = await this.evaluateFunction(tabId, `function (sel, actionDeadlineAt, pageGuard) {+ ${PAGE_AGENT_DOM_ACTION_SOURCE}const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;const deadlineFailure = () => ({ success: false, dispatched: false, noDispatch: true, deadlineExpired: true, error: 'Click action deadline expired' });if (deadlineExpired()) return deadlineFailure();- const sel = ${selectorJSON};const queryDeep = (root) => {try { const h = root.querySelector(sel); if (h) return h; } catch (e) { return null; }const w = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);@@ -4618,12 +4646,19 @@ export class CDPClient {? type === 'submit' || (!type && !!(el.form || el.closest?.('form'))): false;if (deadlineExpired()) return deadlineFailure();+ if (!beforePageAgentDomAction(pageGuard, 'focus'))+ return { success: false, pageFeedbackPending: true, noDispatch: true, dispatched: false };try { el.focus(); } catch (e) {}if (deadlineExpired()) return deadlineFailure();+ if (!el.isConnected)+ return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'Click target changed during focus.' };+ if (!beforePageAgentDomAction(pageGuard, 'click'))+ return { success: false, pageFeedbackPending: true, noDispatch: false, dispatched: true };el.click();const r = el.getBoundingClientRect();return {success: true,+ dispatched: true,method: 'js-click',tag,type,@@ -4631,9 +4666,13 @@ export class CDPClient {text: (el.innerText || '').slice(0, 80),rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },};- })()- `);+ }`, [selector, deadlineAt, pageGuard || null], { timeoutMs: deadlineAt > 0 ? Math.max(1, Math.min(15000, deadlineAt - Date.now())) : 15000 });const fallbackResult = fb?.result?.value || { success: false, error: 'Click failed' };+ if (fallbackResult.pageFeedbackPending === true) {+ await this.consumeFileInputClickGuard(tabId);+ return pageFeedbackClickResult(priorDispatchAttempted, fallbackResult.dispatched === true);+ }+ if (fallbackResult.outcomeUnknown === true && fallbackResult.dispatched === true) return fallbackResult;if (fallbackResult.deadlineExpired === true && fallbackResult.dispatched !== true) {return pageDeadlineResult(priorDispatchAttempted, fallbackResult.error);}diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex c96b55154..964ac15ee 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -363,6 +363,7 @@el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),el.getAttribute('aria-pressed'),+ el.getAttribute('type'),el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),el.getAttribute('formaction'), el.getAttribute('formtarget'),el.getAttribute('popover'), popoverOpen(el),@@ -437,7 +438,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',- 'href', 'target', 'download', 'action', 'formaction', 'formtarget',+ 'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {const el = elementFor(event);@@ -629,14 +630,24 @@try {const guard = JSON.parse(String(event.detail));const op = operations.get(guard.operationId);+ const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const actionPhase = phase !== 'dom';+ const phaseAlreadyConsumed = phase === 'dom'+ ? op?.domDispatchConsumed === true+ : op?.domDispatchConsumed === true || op?.domDispatchPhases?.has(phase) === true;event.stopImmediatePropagation();flushPendingMutations();sampleFormControls(op?.target);if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision- || !op || op.domDispatchConsumed || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {+ || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')+ || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {event.preventDefault(); return;}- op.domDispatchConsumed = true;+ if (actionPhase) {+ op.domDispatchPhases ||= new Set();+ op.domDispatchPhases.add(phase);+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });+ } else op.domDispatchConsumed = true;const marker = { userAt: lastUserAt }; agentTurn = marker;setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex c96b55154..964ac15ee 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -363,6 +363,7 @@el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),el.getAttribute('aria-pressed'),+ el.getAttribute('type'),el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),el.getAttribute('formaction'), el.getAttribute('formtarget'),el.getAttribute('popover'), popoverOpen(el),@@ -437,7 +438,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',- 'href', 'target', 'download', 'action', 'formaction', 'formtarget',+ 'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {const el = elementFor(event);@@ -629,14 +630,24 @@try {const guard = JSON.parse(String(event.detail));const op = operations.get(guard.operationId);+ const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const actionPhase = phase !== 'dom';+ const phaseAlreadyConsumed = phase === 'dom'+ ? op?.domDispatchConsumed === true+ : op?.domDispatchConsumed === true || op?.domDispatchPhases?.has(phase) === true;event.stopImmediatePropagation();flushPendingMutations();sampleFormControls(op?.target);if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision- || !op || op.domDispatchConsumed || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {+ || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')+ || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {event.preventDefault(); return;}- op.domDispatchConsumed = true;+ if (actionPhase) {+ op.domDispatchPhases ||= new Set();+ op.domDispatchPhases.add(phase);+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });+ } else op.domDispatchConsumed = true;const marker = { userAt: lastUserAt }; agentTurn = marker;setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 87fee5e2b..1e24b1f5a 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -135,6 +135,72 @@ for (const build of ['chrome', 'firefox']) {agent._releaseRunEntry(tab);}});++ test('Chrome CDP click fallbacks pass the monitor guard and skip stale DOM clicks', async () => {+ const tab = nextTab++;+ const previousSendMessage = api.tabs.sendMessage;+ const responses = [];+ const { pageFeedbackMethods } = await import('../src/chrome/src/agent/page-feedback.js');+ class ClickAgent {+ static STATE_CHANGE_TOOLS = new Set(['click']);+ isRunning() { return true; }+ _checkAbort() { return false; }+ _persist() {}+ }+ Object.assign(ClickAgent.prototype, pageFeedbackMethods);+ const agent = new ClickAgent();+ api.tabs.sendMessage = async (_tab, message) => {+ if (message.action === 'page_monitor_dispatch') {+ const guard = { runToken: message.params.runToken, documentToken: 'click-document',+ revision: 9, operationId: message.params.operationId };+ responses.push(guard);+ return { ready: true, guard };+ }+ return { ready: true };+ };+ await agent._beginPageFeedbackRun(tab, 'interactive');+ try {+ for (const fallback of ['closed-shadow-node', 'open-shadow-selector']) {+ const client = new CDPClient(), declarationCalls = [];+ client.resolveSelector = async () => ({ inViewport: false, hitOk: false, nodeId: fallback === 'closed-shadow-node' ? 42 : null,+ tag: 'BUTTON', x: 20, y: 30, width: 40, height: 20, text: 'Save' });+ client.armFileInputClickGuard = async () => {};+ client.consumeFileInputClickGuard = async () => ({ blocked: false });+ client.sendCommand = async (_tab, method, params) => {+ if (method === 'DOM.resolveNode') return { object: { objectId: 'click-target' } };+ if (method === 'Runtime.callFunctionOn') {+ declarationCalls.push({ method, params });+ return { result: { value: { pageFeedbackPending: true, noDispatch: true, dispatched: false } } };+ }+ return {};+ };+ client.evaluateFunction = async (_tab, functionDeclaration, args) => {+ declarationCalls.push({ method: 'Runtime.callFunctionOn-global', functionDeclaration, args });+ return { result: { value: { pageFeedbackPending: true, noDispatch: true, dispatched: false } } };+ };+ client.evaluate = async () => { throw new Error('Fallback must pass guarded arguments, not build Runtime.evaluate source'); };+ const result = await client.clickElement(tab, '#target', { beforeDispatch: async () => ({ success: true }) });+ assert.equal(result.pageFeedbackPending, true);+ assert.equal(result.noDispatch, true);+ assert.equal(result.dispatched, false);+ const call = declarationCalls[0];+ const functionDeclaration = call.functionDeclaration || call.params.functionDeclaration;+ assert.match(functionDeclaration, /beforePageAgentDomAction\(pageGuard, 'focus'\)/);+ assert.match(functionDeclaration, /beforePageAgentDomAction\(pageGuard, 'click'\)/);+ const focusCall = functionDeclaration.includes('this.focus()') ? 'this.focus()' : 'el.focus()';+ const clickCall = functionDeclaration.includes('this.click()') ? 'this.click()' : 'el.click()';+ assert.ok(functionDeclaration.lastIndexOf("beforePageAgentDomAction(pageGuard, 'focus')") < functionDeclaration.indexOf(focusCall));+ assert.ok(functionDeclaration.lastIndexOf("beforePageAgentDomAction(pageGuard, 'click')") < functionDeclaration.indexOf(clickCall));+ const passedGuard = call.args+ ? call.args.at(-1)+ : call.params.arguments.at(-1).value;+ assert.deepEqual(passedGuard, responses.at(-1));+ }+ } finally {+ agent._finishPageFeedbackRun(tab);+ api.tabs.sendMessage = previousSendMessage;+ }+ });}test(`${build}: observations are isolated by run, frame, document and sequence`, async () => {diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex ab73838ea..73b243574 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1532,6 +1532,32 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: button type changes invalidate a prepared click`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const form = document.createElement('form');+ form.innerHTML = '<button id="guarded-submit" type="button">Save</button>';+ form.addEventListener('submit', event => { event.preventDefault(); window.formWasSubmitted = true; });+ document.body.append(form);+ });+ await page.waitForTimeout(150);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'button-type', tool: 'click', selector: '#guarded-submit' });+ deliver('page_monitor_dispatch', { operationId: 'button-type', kind: 'click', selector: '#guarded-submit' });+ return lastMonitorResponse.guard;+ });+ await page.locator('#guarded-submit').evaluate(element => element.setAttribute('type', 'submit'));+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#guarded-submit'));+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, 'A type mutation must make the prepared focus/click guard stale');+ assert.equal(await page.evaluate(() => window.formWasSubmitted === true), false);+ } finally { await browser.close(); }+ });+for (const action of ['click', 'type']) {test(`${build}: ${action} preparation preserves same-target human input`, async () => {const { browser, page } = await fixture(engine, build);@@ -1849,6 +1875,33 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: guarded DOM-click fallback gates focus and click once each`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'dom-click-phases', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'dom-click-phases', kind: 'click', selector: '#agent' });+ return lastMonitorResponse.guard;+ });+ const phases = await page.evaluate(value => {+ const dispatch = phase => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: phase }), cancelable: true,+ }));+ const focus = dispatch('focus');+ document.getElementById('agent').focus();+ const click = dispatch('click');+ if (click) document.getElementById('agent').click();+ const replay = dispatch('click');+ return { focus, click, replay };+ }, guard);+ assert.deepEqual(phases, { focus: true, click: true, replay: false });+ assert.equal(await page.locator('#status').textContent(), 'Agent changed this');+ await page.waitForTimeout(150);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);+ } finally { await browser.close(); }+ });+test(`${build}: explicit agent focus is attributed before focusin`, async () => {const { browser, page } = await fixture(engine, build);try {