Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard CDP click fallbacks against page feedback

Barack Sokullu12b61a596c7d
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index efa20c080..0b00685fa 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -39890,8 +39890,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
return { success: true };
},
});
- if (selResult?.deadlineExpired === true && selResult?.dispatched !== true) {
+ if ((selResult?.noDispatch === true || selResult?.dispatched === false) && selResult?.outcomeUnknown !== true) {
earlyCdpDispatchState.started = false;
+ }
+ if (selResult?.deadlineExpired === true && selResult?.dispatched !== true) {
return {
...selResult,
success: false,
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index 4dc89cb79..eefe4074c 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -18,6 +18,16 @@ const SELECTOR_SCROLL_DISPATCH_SOURCE = `
};
`;
+const PAGE_AGENT_DOM_ACTION_SOURCE = `
+ const beforePageAgentDomAction = (guard, phase) => {
+ if (!guard) return true;
+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...guard, dispatchPhase: phase }), cancelable: true,
+ });
+ return window.dispatchEvent(event);
+ };
+`;
+
function throwIfPageFeedbackPending(result) {
if (!result?.pageFeedbackPending) return;
const error = new Error(pageFeedbackPendingResult().error);
@@ -4444,6 +4454,17 @@ export class CDPClient {
deadlineExpired: true,
error: error || 'Click action deadline expired before click dispatch',
};
+ const pageFeedbackClickResult = (priorDispatchAttempted, focusDispatched = false) => {
+ if (!priorDispatchAttempted && !focusDispatched) return pageFeedbackPendingResult();
+ return {
+ success: false,
+ dispatched: true,
+ outcomeUnknown: true,
+ retryable: false,
+ pageFeedbackPending: true,
+ error: 'The page changed during click preparation. A prior click or focus may have taken effect; inspect the page before retrying.',
+ };
+ };
if (info.inViewport && info.hitOk) {
try {
await this.armFileInputClickGuard(tabId);
@@ -4534,21 +4555,31 @@ export class CDPClient {
const priorDispatchAttempted = dispatchAttempted;
const validation = await authorizeDispatch({ x: info.x, y: info.y, tag: info.tag });
if (validation.success !== true) return validation;
- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });
+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });
dispatchAttempted = true;
const clicked = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {
objectId,
- functionDeclaration: `function(actionDeadlineAt) {
+ functionDeclaration: `function(actionDeadlineAt, pageGuard) {
+ ${PAGE_AGENT_DOM_ACTION_SOURCE}
if (actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt) return false;
+ if (!this || !this.isConnected) return false;
+ if (!beforePageAgentDomAction(pageGuard, 'focus'))
+ return { pageFeedbackPending: true, noDispatch: true, dispatched: false };
try { this.focus(); } catch {}
if (actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt) return false;
+ if (!this.isConnected || !beforePageAgentDomAction(pageGuard, 'click'))
+ return { pageFeedbackPending: true, noDispatch: false, dispatched: true };
this.click();
return true;
}`,
- arguments: [{ value: deadlineAt }],
+ arguments: [{ value: deadlineAt }, { value: pageGuard || null }],
returnByValue: true,
awaitPromise: false,
});
+ if (clicked?.result?.value?.pageFeedbackPending === true) {
+ await this.consumeFileInputClickGuard(tabId);
+ return pageFeedbackClickResult(priorDispatchAttempted, clicked.result.value.dispatched === true);
+ }
if (clicked?.result?.value === false) {
return pageDeadlineResult(priorDispatchAttempted, 'Click action deadline expired before click dispatch');
}
@@ -4585,22 +4616,19 @@ export class CDPClient {
}
// Step 3: JS fallback for open shadow roots.
- const selectorJSON = JSON.stringify(selector);
throwIfAborted();
await this.armFileInputClickGuard(tabId);
throwIfAborted();
const fallbackValidation = await authorizeDispatch({ x: info.x, y: info.y, tag: info.tag });
if (fallbackValidation.success !== true) return fallbackValidation;
const priorDispatchAttempted = dispatchAttempted;
- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });
+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'click', selector, x: info.x, y: info.y });
dispatchAttempted = true;
- const fb = await this.evaluate(tabId, `
- (() => {
- const actionDeadlineAt = ${deadlineAt};
+ const fb = await this.evaluateFunction(tabId, `function (sel, actionDeadlineAt, pageGuard) {
+ ${PAGE_AGENT_DOM_ACTION_SOURCE}
const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;
const deadlineFailure = () => ({ success: false, dispatched: false, noDispatch: true, deadlineExpired: true, error: 'Click action deadline expired' });
if (deadlineExpired()) return deadlineFailure();
- const sel = ${selectorJSON};
const queryDeep = (root) => {
try { const h = root.querySelector(sel); if (h) return h; } catch (e) { return null; }
const w = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT);
@@ -4618,12 +4646,19 @@ export class CDPClient {
? type === 'submit' || (!type && !!(el.form || el.closest?.('form')))
: false;
if (deadlineExpired()) return deadlineFailure();
+ if (!beforePageAgentDomAction(pageGuard, 'focus'))
+ return { success: false, pageFeedbackPending: true, noDispatch: true, dispatched: false };
try { el.focus(); } catch (e) {}
if (deadlineExpired()) return deadlineFailure();
+ if (!el.isConnected)
+ return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'Click target changed during focus.' };
+ if (!beforePageAgentDomAction(pageGuard, 'click'))
+ return { success: false, pageFeedbackPending: true, noDispatch: false, dispatched: true };
el.click();
const r = el.getBoundingClientRect();
return {
success: true,
+ dispatched: true,
method: 'js-click',
tag,
type,
@@ -4631,9 +4666,13 @@ export class CDPClient {
text: (el.innerText || '').slice(0, 80),
rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },
};
- })()
- `);
+ }`, [selector, deadlineAt, pageGuard || null], { timeoutMs: deadlineAt > 0 ? Math.max(1, Math.min(15000, deadlineAt - Date.now())) : 15000 });
const fallbackResult = fb?.result?.value || { success: false, error: 'Click failed' };
+ if (fallbackResult.pageFeedbackPending === true) {
+ await this.consumeFileInputClickGuard(tabId);
+ return pageFeedbackClickResult(priorDispatchAttempted, fallbackResult.dispatched === true);
+ }
+ if (fallbackResult.outcomeUnknown === true && fallbackResult.dispatched === true) return fallbackResult;
if (fallbackResult.deadlineExpired === true && fallbackResult.dispatched !== true) {
return pageDeadlineResult(priorDispatchAttempted, fallbackResult.error);
}
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index c96b55154..964ac15ee 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -363,6 +363,7 @@
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),
el.getAttribute('aria-pressed'),
+ el.getAttribute('type'),
el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),
el.getAttribute('formaction'), el.getAttribute('formtarget'),
el.getAttribute('popover'), popoverOpen(el),
@@ -437,7 +438,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
- 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
+ 'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
@@ -629,14 +630,24 @@
try {
const guard = JSON.parse(String(event.detail));
const op = operations.get(guard.operationId);
+ const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const actionPhase = phase !== 'dom';
+ const phaseAlreadyConsumed = phase === 'dom'
+ ? op?.domDispatchConsumed === true
+ : op?.domDispatchConsumed === true || op?.domDispatchPhases?.has(phase) === true;
event.stopImmediatePropagation();
flushPendingMutations();
sampleFormControls(op?.target);
if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision
- || !op || op.domDispatchConsumed || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
+ || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')
+ || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
event.preventDefault(); return;
}
- op.domDispatchConsumed = true;
+ if (actionPhase) {
+ op.domDispatchPhases ||= new Set();
+ op.domDispatchPhases.add(phase);
+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });
+ } else op.domDispatchConsumed = true;
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index c96b55154..964ac15ee 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -363,6 +363,7 @@
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-valuenow'), el.getAttribute('aria-valuetext'),
el.getAttribute('aria-pressed'),
+ el.getAttribute('type'),
el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),
el.getAttribute('formaction'), el.getAttribute('formtarget'),
el.getAttribute('popover'), popoverOpen(el),
@@ -437,7 +438,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
- 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
+ 'type', 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-valuenow', 'aria-valuetext', 'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
@@ -629,14 +630,24 @@
try {
const guard = JSON.parse(String(event.detail));
const op = operations.get(guard.operationId);
+ const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const actionPhase = phase !== 'dom';
+ const phaseAlreadyConsumed = phase === 'dom'
+ ? op?.domDispatchConsumed === true
+ : op?.domDispatchConsumed === true || op?.domDispatchPhases?.has(phase) === true;
event.stopImmediatePropagation();
flushPendingMutations();
sampleFormControls(op?.target);
if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision
- || !op || op.domDispatchConsumed || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
+ || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')
+ || op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
event.preventDefault(); return;
}
- op.domDispatchConsumed = true;
+ if (actionPhase) {
+ op.domDispatchPhases ||= new Set();
+ op.domDispatchPhases.add(phase);
+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });
+ } else op.domDispatchConsumed = true;
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 87fee5e2b..1e24b1f5a 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -135,6 +135,72 @@ for (const build of ['chrome', 'firefox']) {
agent._releaseRunEntry(tab);
}
});
+
+ test('Chrome CDP click fallbacks pass the monitor guard and skip stale DOM clicks', async () => {
+ const tab = nextTab++;
+ const previousSendMessage = api.tabs.sendMessage;
+ const responses = [];
+ const { pageFeedbackMethods } = await import('../src/chrome/src/agent/page-feedback.js');
+ class ClickAgent {
+ static STATE_CHANGE_TOOLS = new Set(['click']);
+ isRunning() { return true; }
+ _checkAbort() { return false; }
+ _persist() {}
+ }
+ Object.assign(ClickAgent.prototype, pageFeedbackMethods);
+ const agent = new ClickAgent();
+ api.tabs.sendMessage = async (_tab, message) => {
+ if (message.action === 'page_monitor_dispatch') {
+ const guard = { runToken: message.params.runToken, documentToken: 'click-document',
+ revision: 9, operationId: message.params.operationId };
+ responses.push(guard);
+ return { ready: true, guard };
+ }
+ return { ready: true };
+ };
+ await agent._beginPageFeedbackRun(tab, 'interactive');
+ try {
+ for (const fallback of ['closed-shadow-node', 'open-shadow-selector']) {
+ const client = new CDPClient(), declarationCalls = [];
+ client.resolveSelector = async () => ({ inViewport: false, hitOk: false, nodeId: fallback === 'closed-shadow-node' ? 42 : null,
+ tag: 'BUTTON', x: 20, y: 30, width: 40, height: 20, text: 'Save' });
+ client.armFileInputClickGuard = async () => {};
+ client.consumeFileInputClickGuard = async () => ({ blocked: false });
+ client.sendCommand = async (_tab, method, params) => {
+ if (method === 'DOM.resolveNode') return { object: { objectId: 'click-target' } };
+ if (method === 'Runtime.callFunctionOn') {
+ declarationCalls.push({ method, params });
+ return { result: { value: { pageFeedbackPending: true, noDispatch: true, dispatched: false } } };
+ }
+ return {};
+ };
+ client.evaluateFunction = async (_tab, functionDeclaration, args) => {
+ declarationCalls.push({ method: 'Runtime.callFunctionOn-global', functionDeclaration, args });
+ return { result: { value: { pageFeedbackPending: true, noDispatch: true, dispatched: false } } };
+ };
+ client.evaluate = async () => { throw new Error('Fallback must pass guarded arguments, not build Runtime.evaluate source'); };
+ const result = await client.clickElement(tab, '#target', { beforeDispatch: async () => ({ success: true }) });
+ assert.equal(result.pageFeedbackPending, true);
+ assert.equal(result.noDispatch, true);
+ assert.equal(result.dispatched, false);
+ const call = declarationCalls[0];
+ const functionDeclaration = call.functionDeclaration || call.params.functionDeclaration;
+ assert.match(functionDeclaration, /beforePageAgentDomAction\(pageGuard, 'focus'\)/);
+ assert.match(functionDeclaration, /beforePageAgentDomAction\(pageGuard, 'click'\)/);
+ const focusCall = functionDeclaration.includes('this.focus()') ? 'this.focus()' : 'el.focus()';
+ const clickCall = functionDeclaration.includes('this.click()') ? 'this.click()' : 'el.click()';
+ assert.ok(functionDeclaration.lastIndexOf("beforePageAgentDomAction(pageGuard, 'focus')") < functionDeclaration.indexOf(focusCall));
+ assert.ok(functionDeclaration.lastIndexOf("beforePageAgentDomAction(pageGuard, 'click')") < functionDeclaration.indexOf(clickCall));
+ const passedGuard = call.args
+ ? call.args.at(-1)
+ : call.params.arguments.at(-1).value;
+ assert.deepEqual(passedGuard, responses.at(-1));
+ }
+ } finally {
+ agent._finishPageFeedbackRun(tab);
+ api.tabs.sendMessage = previousSendMessage;
+ }
+ });
}
test(`${build}: observations are isolated by run, frame, document and sequence`, async () => {
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index ab73838ea..73b243574 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1532,6 +1532,32 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: button type changes invalidate a prepared click`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const form = document.createElement('form');
+ form.innerHTML = '<button id="guarded-submit" type="button">Save</button>';
+ form.addEventListener('submit', event => { event.preventDefault(); window.formWasSubmitted = true; });
+ document.body.append(form);
+ });
+ await page.waitForTimeout(150);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'button-type', tool: 'click', selector: '#guarded-submit' });
+ deliver('page_monitor_dispatch', { operationId: 'button-type', kind: 'click', selector: '#guarded-submit' });
+ return lastMonitorResponse.guard;
+ });
+ await page.locator('#guarded-submit').evaluate(element => element.setAttribute('type', 'submit'));
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'button#guarded-submit'));
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, 'A type mutation must make the prepared focus/click guard stale');
+ assert.equal(await page.evaluate(() => window.formWasSubmitted === true), false);
+ } finally { await browser.close(); }
+ });
+
for (const action of ['click', 'type']) {
test(`${build}: ${action} preparation preserves same-target human input`, async () => {
const { browser, page } = await fixture(engine, build);
@@ -1849,6 +1875,33 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: guarded DOM-click fallback gates focus and click once each`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'dom-click-phases', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'dom-click-phases', kind: 'click', selector: '#agent' });
+ return lastMonitorResponse.guard;
+ });
+ const phases = await page.evaluate(value => {
+ const dispatch = phase => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: phase }), cancelable: true,
+ }));
+ const focus = dispatch('focus');
+ document.getElementById('agent').focus();
+ const click = dispatch('click');
+ if (click) document.getElementById('agent').click();
+ const replay = dispatch('click');
+ return { focus, click, replay };
+ }, guard);
+ assert.deepEqual(phases, { focus: true, click: true, replay: false });
+ assert.equal(await page.locator('#status').textContent(), 'Agent changed this');
+ await page.waitForTimeout(150);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);
+ } finally { await browser.close(); }
+ });
+
test(`${build}: explicit agent focus is attributed before focusin`, async () => {
const { browser, page } = await fixture(engine, build);
try {