Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix native hover attribution and guarded JS evaluation

Barack Sokullub0db388af1ff
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 29fdaf8b8..efa20c080 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -33565,13 +33565,16 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
// Match Firefox's function-body contract while adding async/await:
// callers use an explicit `return` for readback instead of having to
// squeeze a multi-statement edit into one JavaScript expression.
- let expression = `(async () => {\n${code}\n})()`;
const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'dom' });
const abortedDispatchToken = pageGuard ? `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}` : null;
- if (pageGuard) expression = `(() => { const gate = new CustomEvent('webbrain-agent-dom-dispatch', { detail: ${JSON.stringify(JSON.stringify(pageGuard))}, cancelable: true }); if (!window.dispatchEvent(gate)) return { __webbrainPageFeedbackAborted: ${JSON.stringify(abortedDispatchToken)} }; return ${expression}; })()`;
- expression += '\n//# sourceURL=webbrain-dev-execute.js';
dispatched = true;
- const response = await cdpClient.evaluate(tabId, expression, true, { timeoutMs: 15000 });
+ const response = pageGuard
+ ? await cdpClient.evaluateFunction(tabId, `function (guard, abortedToken) {
+ const gate = new CustomEvent('webbrain-agent-dom-dispatch', { detail: JSON.stringify(guard), cancelable: true });
+ if (!window.dispatchEvent(gate)) return { __webbrainPageFeedbackAborted: abortedToken };
+ return (async () => {\n${code}\n})();
+ }\n//# sourceURL=webbrain-dev-execute.js`, [pageGuard, abortedDispatchToken], { timeoutMs: 15000 })
+ : await cdpClient.evaluate(tabId, `(async () => {\n${code}\n})()\n//# sourceURL=webbrain-dev-execute.js`, true, { timeoutMs: 15000 });
if (response?.exceptionDetails) {
const details = response.exceptionDetails;
const exception = details.exception || {};
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index cb2c6d09e..4dc89cb79 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -2254,6 +2254,36 @@ export class CDPClient {
return result;
}
+ /** Evaluate a function on the page global, passing data as protocol arguments. */
+ async evaluateFunction(tabId, functionDeclaration, args = [], options = {}) {
+ await this.sendCommand(tabId, 'Runtime.enable');
+ const global = await this.sendCommand(tabId, 'Runtime.evaluate', {
+ expression: 'globalThis',
+ returnByValue: false,
+ userGesture: true,
+ });
+ const objectId = global?.result?.objectId;
+ if (!objectId) return global;
+ const requestedTimeout = Number(options?.timeoutMs);
+ const params = {
+ functionDeclaration,
+ objectId,
+ arguments: args.map(value => ({ value })),
+ returnByValue: true,
+ awaitPromise: true,
+ userGesture: true,
+ allowUnsafeEvalBlockedByCSP: true,
+ };
+ if (Number.isFinite(requestedTimeout) && requestedTimeout > 0) {
+ params.timeout = Math.max(1, Math.min(30000, Math.round(requestedTimeout)));
+ }
+ try {
+ return await this.sendCommand(tabId, 'Runtime.callFunctionOn', params);
+ } finally {
+ await this.sendCommand(tabId, 'Runtime.releaseObject', { objectId }).catch(() => {});
+ }
+ }
+
/**
* Call function on an object.
*/
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 138a401fa..c96b55154 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -231,7 +231,8 @@
send({ kind: 'dom', source, target: targetName(target || currentHit) });
return true;
}
- function expected(kind, el, event) {
+ function expected(kind, el, event, nativeMarkerOnly = false) {
+ let activatedNativeMarker = false;
const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))
|| (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);
const marker = markerNode?.getAttribute(nativeMarkerAttribute);
@@ -243,7 +244,7 @@
const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;
const allowedEvent = data.kind === 'input'
? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
- : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ : data.kind === 'click' ? ['pointerover', 'mouseover', 'pointermove', 'pointerdown', 'mousedown', 'click'].includes(event.type) : false;
const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision
&& !domTimer && !unreported && lastUserAt <= op.userAt;
const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
@@ -253,9 +254,14 @@
op.nativeSequence = data.sequence;
markerNode.removeAttribute(nativeMarkerAttribute);
dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });
+ activatedNativeMarker = true;
+ const markerTurn = { userAt: op.userAt };
+ agentTurn = markerTurn;
+ setTimeout(() => { if (agentTurn === markerTurn) agentTurn = null; }, 0);
}
} catch { /* Page markers are hints; malformed ones grant no expectation. */ }
}
+ if (nativeMarkerOnly && !activatedNativeMarker) return null;
if (event && matchedEvents.has(event)) return matchedEvents.get(event);
prune();
const match = op => {
@@ -666,6 +672,9 @@
// exact input's attribution through its page handlers, until the next task.
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
};
+ for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {
+ if (event.isTrusted) expected('click', elementFor(event), event, true);
+ });
for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);
listen(document, 'pointerdown', event => {
const el = elementFor(event);
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 138a401fa..c96b55154 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -231,7 +231,8 @@
send({ kind: 'dom', source, target: targetName(target || currentHit) });
return true;
}
- function expected(kind, el, event) {
+ function expected(kind, el, event, nativeMarkerOnly = false) {
+ let activatedNativeMarker = false;
const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))
|| (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);
const marker = markerNode?.getAttribute(nativeMarkerAttribute);
@@ -243,7 +244,7 @@
const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;
const allowedEvent = data.kind === 'input'
? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
- : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ : data.kind === 'click' ? ['pointerover', 'mouseover', 'pointermove', 'pointerdown', 'mousedown', 'click'].includes(event.type) : false;
const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision
&& !domTimer && !unreported && lastUserAt <= op.userAt;
const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
@@ -253,9 +254,14 @@
op.nativeSequence = data.sequence;
markerNode.removeAttribute(nativeMarkerAttribute);
dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });
+ activatedNativeMarker = true;
+ const markerTurn = { userAt: op.userAt };
+ agentTurn = markerTurn;
+ setTimeout(() => { if (agentTurn === markerTurn) agentTurn = null; }, 0);
}
} catch { /* Page markers are hints; malformed ones grant no expectation. */ }
}
+ if (nativeMarkerOnly && !activatedNativeMarker) return null;
if (event && matchedEvents.has(event)) return matchedEvents.get(event);
prune();
const match = op => {
@@ -666,6 +672,9 @@
// exact input's attribution through its page handlers, until the next task.
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
};
+ for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {
+ if (event.isTrusted) expected('click', elementFor(event), event, true);
+ });
for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);
listen(document, 'pointerdown', event => {
const el = elementFor(event);
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 6bedd6acf..87fee5e2b 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -3,7 +3,7 @@ import fs from 'node:fs';
import vm from 'node:vm';
import { test } from 'node:test';
import { makeSchedulerHarness } from './lib/scheduler-harness.mjs';
-import { cdpClient } from '../src/chrome/src/cdp/cdp-client.js';
+import { CDPClient, cdpClient } from '../src/chrome/src/cdp/cdp-client.js';
const area = { get: async () => ({}), set: async () => {}, remove: async () => {} };
const api = { storage: { local: area, session: area },
@@ -16,6 +16,29 @@ const deferred = () => { let resolve; const promise = new Promise(r => { resolve
const sender = (tabId, frameId = 0, documentId = 'document-1') => ({ tab: { id: tabId }, frameId, documentId, url: 'https://example.com/new' });
let nextTab = 600;
+test('CDP evaluateFunction sends page data as arguments and releases the global target', async () => {
+ const client = new CDPClient(), tab = 599, calls = [];
+ client.sessions.set(tab, {});
+ client.sendCommand = async (_tab, method, params) => {
+ calls.push({ method, params });
+ if (method === 'Runtime.evaluate') return { result: { objectId: 'page-global' } };
+ if (method === 'Runtime.callFunctionOn') return { result: { value: 'done' } };
+ return {};
+ };
+ const guard = { documentToken: 'doc', revision: 3, operationId: 'op' };
+ const declaration = 'function (guard, token) { return [guard, token]; }';
+ const response = await client.evaluateFunction(tab, declaration, [guard, 'abort'], { timeoutMs: 15000 });
+ assert.deepEqual(response, { result: { value: 'done' } });
+ assert.deepEqual(calls.map(call => call.method), ['Runtime.enable', 'Runtime.evaluate', 'Runtime.callFunctionOn', 'Runtime.releaseObject']);
+ assert.equal(calls[1].params.expression, 'globalThis');
+ assert.equal(calls[2].params.objectId, 'page-global');
+ assert.equal(calls[2].params.functionDeclaration, declaration);
+ assert.deepEqual(calls[2].params.arguments, [{ value: guard }, { value: 'abort' }]);
+ assert.equal(calls[2].params.awaitPromise, true);
+ assert.equal(calls[2].params.timeout, 15000);
+ assert.deepEqual(calls[3].params, { objectId: 'page-global' });
+});
+
function setup(Agent, implementation = {}) {
const provider = { name: 'feedback test', model: 'test', promptTier: 'full', contextWindow: 128000,
supportsTools: false, supportsVision: false, ...implementation };
@@ -72,7 +95,8 @@ for (const build of ['chrome', 'firefox']) {
const tab = nextTab++, agent = setup(Agent);
const previousSendMessage = api.tabs.sendMessage;
const previousEnable = cdpClient.enableDevDiagnostics, previousEvaluate = cdpClient.evaluate;
- let expression = '';
+ const previousEvaluateFunction = cdpClient.evaluateFunction;
+ let declaration = '', passedArgs = [];
try {
await agent._claimRunEntry(tab, 'interactive');
await agent._beginPageFeedbackRun(tab, 'interactive');
@@ -83,22 +107,29 @@ for (const build of ['chrome', 'firefox']) {
revision: 7, operationId: message.params.operationId } }
: { ready: true };
cdpClient.enableDevDiagnostics = async () => {};
- cdpClient.evaluate = async (_tab, value) => {
- expression = value;
- const token = value.match(/__webbrainPageFeedbackAborted: "([^"]+)"/)?.[1];
- assert.ok(token, 'The wrapped page expression must have a unique aborted-dispatch result');
- return { result: { value: { __webbrainPageFeedbackAborted: token } } };
+ cdpClient.evaluateFunction = async (_tab, value, args) => {
+ declaration = value; passedArgs = args;
+ assert.equal(typeof args[0], 'object');
+ assert.equal(typeof args[1], 'string');
+ return { result: { value: { __webbrainPageFeedbackAborted: args[1] } } };
};
const result = await agent._executeDevJavaScript(tab, { code: 'window.__shouldNotRun = true;' });
assert.equal(result.dispatched, false);
assert.equal(result.noDispatch, true);
assert.match(result.error, /page changed during JavaScript preparation/i);
- assert.match(expression, /cancelable: true/);
- assert.match(expression, /if \(!window\.dispatchEvent\(gate\)\) return/);
- assert.ok(expression.indexOf('return (async () =>') > expression.indexOf('dispatchEvent(gate)'));
+ assert.match(declaration, /cancelable: true/);
+ assert.match(declaration, /if \(!window\.dispatchEvent\(gate\)\) return/);
+ assert.ok(declaration.indexOf('return (async () =>') > declaration.indexOf('dispatchEvent(gate)'));
+ assert.match(declaration, /window\.__shouldNotRun = true/);
+ assert.doesNotMatch(declaration, /execute-js-document|operationId/,
+ 'The page guard must travel as a protocol argument instead of source text');
+ assert.equal(passedArgs[0].documentToken, 'execute-js-document');
+ assert.ok(passedArgs[1]);
+ assert.equal(cdpClient.evaluate, previousEvaluate, 'Guarded execution must not use Runtime.evaluate source construction');
} finally {
cdpClient.enableDevDiagnostics = previousEnable;
cdpClient.evaluate = previousEvaluate;
+ cdpClient.evaluateFunction = previousEvaluateFunction;
api.tabs.sendMessage = previousSendMessage;
if (agent._pageFeedbackRuns.has(tab)) agent._finishPageFeedbackRun(tab);
agent._releaseRunEntry(tab);
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 7c394c33b..ab73838ea 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1891,6 +1891,35 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: native hover markers attribute hover-triggered page changes to the agent`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ document.getElementById('agent').addEventListener('pointerover', () => {
+ document.getElementById('status').textContent = 'Pointer hover changed';
+ });
+ document.getElementById('agent').addEventListener('mouseover', () => {
+ document.getElementById('status').textContent = 'Mouse hover changed';
+ });
+ deliver('page_monitor_prepare', { operationId: 'native-hover', tool: 'hover', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'native-hover', kind: 'click', navigationCandidate: false, fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ const marker = createNativeActionMarker(guard, 'click', 1);
+ await page.locator('#agent').evaluate((element, value) => element.setAttribute('data-webbrain-native-action', value), marker);
+ await page.locator('#agent').hover();
+ assert.match(await page.locator('#status').textContent(), /hover changed/i);
+ await page.waitForTimeout(200);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,
+ 'The first trusted hover event must activate and attribute its prepared native operation');
+ await page.locator('#human').click();
+ await page.waitForTimeout(200);
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'),
+ 'A subsequent user click remains visible to the monitor');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: native uploads target an unfocused file input without hiding user input`, async () => {
const { browser, page } = await fixture(engine, build);
const session = new BidiSession(), runId = crypto.randomUUID(), token = crypto.randomUUID();