Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix native hover attribution and guarded JS evaluation
Barack Sokullub0db388af1ff
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 29fdaf8b8..efa20c080 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -33565,13 +33565,16 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai// Match Firefox's function-body contract while adding async/await:// callers use an explicit `return` for readback instead of having to// squeeze a multi-statement edit into one JavaScript expression.- let expression = `(async () => {\n${code}\n})()`;const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'dom' });const abortedDispatchToken = pageGuard ? `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}` : null;- if (pageGuard) expression = `(() => { const gate = new CustomEvent('webbrain-agent-dom-dispatch', { detail: ${JSON.stringify(JSON.stringify(pageGuard))}, cancelable: true }); if (!window.dispatchEvent(gate)) return { __webbrainPageFeedbackAborted: ${JSON.stringify(abortedDispatchToken)} }; return ${expression}; })()`;- expression += '\n//# sourceURL=webbrain-dev-execute.js';dispatched = true;- const response = await cdpClient.evaluate(tabId, expression, true, { timeoutMs: 15000 });+ const response = pageGuard+ ? await cdpClient.evaluateFunction(tabId, `function (guard, abortedToken) {+ const gate = new CustomEvent('webbrain-agent-dom-dispatch', { detail: JSON.stringify(guard), cancelable: true });+ if (!window.dispatchEvent(gate)) return { __webbrainPageFeedbackAborted: abortedToken };+ return (async () => {\n${code}\n})();+ }\n//# sourceURL=webbrain-dev-execute.js`, [pageGuard, abortedDispatchToken], { timeoutMs: 15000 })+ : await cdpClient.evaluate(tabId, `(async () => {\n${code}\n})()\n//# sourceURL=webbrain-dev-execute.js`, true, { timeoutMs: 15000 });if (response?.exceptionDetails) {const details = response.exceptionDetails;const exception = details.exception || {};diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex cb2c6d09e..4dc89cb79 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -2254,6 +2254,36 @@ export class CDPClient {return result;}+ /** Evaluate a function on the page global, passing data as protocol arguments. */+ async evaluateFunction(tabId, functionDeclaration, args = [], options = {}) {+ await this.sendCommand(tabId, 'Runtime.enable');+ const global = await this.sendCommand(tabId, 'Runtime.evaluate', {+ expression: 'globalThis',+ returnByValue: false,+ userGesture: true,+ });+ const objectId = global?.result?.objectId;+ if (!objectId) return global;+ const requestedTimeout = Number(options?.timeoutMs);+ const params = {+ functionDeclaration,+ objectId,+ arguments: args.map(value => ({ value })),+ returnByValue: true,+ awaitPromise: true,+ userGesture: true,+ allowUnsafeEvalBlockedByCSP: true,+ };+ if (Number.isFinite(requestedTimeout) && requestedTimeout > 0) {+ params.timeout = Math.max(1, Math.min(30000, Math.round(requestedTimeout)));+ }+ try {+ return await this.sendCommand(tabId, 'Runtime.callFunctionOn', params);+ } finally {+ await this.sendCommand(tabId, 'Runtime.releaseObject', { objectId }).catch(() => {});+ }+ }+/*** Call function on an object.*/diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 138a401fa..c96b55154 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -231,7 +231,8 @@send({ kind: 'dom', source, target: targetName(target || currentHit) });return true;}- function expected(kind, el, event) {+ function expected(kind, el, event, nativeMarkerOnly = false) {+ let activatedNativeMarker = false;const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))|| (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);const marker = markerNode?.getAttribute(nativeMarkerAttribute);@@ -243,7 +244,7 @@const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;const allowedEvent = data.kind === 'input'? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)- : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ : data.kind === 'click' ? ['pointerover', 'mouseover', 'pointermove', 'pointerdown', 'mousedown', 'click'].includes(event.type) : false;const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision&& !domTimer && !unreported && lastUserAt <= op.userAt;const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;@@ -253,9 +254,14 @@op.nativeSequence = data.sequence;markerNode.removeAttribute(nativeMarkerAttribute);dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });+ activatedNativeMarker = true;+ const markerTurn = { userAt: op.userAt };+ agentTurn = markerTurn;+ setTimeout(() => { if (agentTurn === markerTurn) agentTurn = null; }, 0);}} catch { /* Page markers are hints; malformed ones grant no expectation. */ }}+ if (nativeMarkerOnly && !activatedNativeMarker) return null;if (event && matchedEvents.has(event)) return matchedEvents.get(event);prune();const match = op => {@@ -666,6 +672,9 @@// exact input's attribution through its page handlers, until the next task.setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);};+ for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {+ if (event.isTrusted) expected('click', elementFor(event), event, true);+ });for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);listen(document, 'pointerdown', event => {const el = elementFor(event);diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 138a401fa..c96b55154 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -231,7 +231,8 @@send({ kind: 'dom', source, target: targetName(target || currentHit) });return true;}- function expected(kind, el, event) {+ function expected(kind, el, event, nativeMarkerOnly = false) {+ let activatedNativeMarker = false;const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))|| (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);const marker = markerNode?.getAttribute(nativeMarkerAttribute);@@ -243,7 +244,7 @@const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;const allowedEvent = data.kind === 'input'? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)- : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ : data.kind === 'click' ? ['pointerover', 'mouseover', 'pointermove', 'pointerdown', 'mousedown', 'click'].includes(event.type) : false;const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision&& !domTimer && !unreported && lastUserAt <= op.userAt;const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;@@ -253,9 +254,14 @@op.nativeSequence = data.sequence;markerNode.removeAttribute(nativeMarkerAttribute);dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });+ activatedNativeMarker = true;+ const markerTurn = { userAt: op.userAt };+ agentTurn = markerTurn;+ setTimeout(() => { if (agentTurn === markerTurn) agentTurn = null; }, 0);}} catch { /* Page markers are hints; malformed ones grant no expectation. */ }}+ if (nativeMarkerOnly && !activatedNativeMarker) return null;if (event && matchedEvents.has(event)) return matchedEvents.get(event);prune();const match = op => {@@ -666,6 +672,9 @@// exact input's attribution through its page handlers, until the next task.setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);};+ for (const name of ['pointerover', 'mouseover']) listen(document, name, event => {+ if (event.isTrusted) expected('click', elementFor(event), event, true);+ });for (const name of ['click', 'input', 'beforeinput', 'change', 'keydown', 'pointerdown', 'pointermove']) listen(document, name, noteAgentTurn);listen(document, 'pointerdown', event => {const el = elementFor(event);diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 6bedd6acf..87fee5e2b 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -3,7 +3,7 @@ import fs from 'node:fs';import vm from 'node:vm';import { test } from 'node:test';import { makeSchedulerHarness } from './lib/scheduler-harness.mjs';-import { cdpClient } from '../src/chrome/src/cdp/cdp-client.js';+import { CDPClient, cdpClient } from '../src/chrome/src/cdp/cdp-client.js';const area = { get: async () => ({}), set: async () => {}, remove: async () => {} };const api = { storage: { local: area, session: area },@@ -16,6 +16,29 @@ const deferred = () => { let resolve; const promise = new Promise(r => { resolveconst sender = (tabId, frameId = 0, documentId = 'document-1') => ({ tab: { id: tabId }, frameId, documentId, url: 'https://example.com/new' });let nextTab = 600;+test('CDP evaluateFunction sends page data as arguments and releases the global target', async () => {+ const client = new CDPClient(), tab = 599, calls = [];+ client.sessions.set(tab, {});+ client.sendCommand = async (_tab, method, params) => {+ calls.push({ method, params });+ if (method === 'Runtime.evaluate') return { result: { objectId: 'page-global' } };+ if (method === 'Runtime.callFunctionOn') return { result: { value: 'done' } };+ return {};+ };+ const guard = { documentToken: 'doc', revision: 3, operationId: 'op' };+ const declaration = 'function (guard, token) { return [guard, token]; }';+ const response = await client.evaluateFunction(tab, declaration, [guard, 'abort'], { timeoutMs: 15000 });+ assert.deepEqual(response, { result: { value: 'done' } });+ assert.deepEqual(calls.map(call => call.method), ['Runtime.enable', 'Runtime.evaluate', 'Runtime.callFunctionOn', 'Runtime.releaseObject']);+ assert.equal(calls[1].params.expression, 'globalThis');+ assert.equal(calls[2].params.objectId, 'page-global');+ assert.equal(calls[2].params.functionDeclaration, declaration);+ assert.deepEqual(calls[2].params.arguments, [{ value: guard }, { value: 'abort' }]);+ assert.equal(calls[2].params.awaitPromise, true);+ assert.equal(calls[2].params.timeout, 15000);+ assert.deepEqual(calls[3].params, { objectId: 'page-global' });+});+function setup(Agent, implementation = {}) {const provider = { name: 'feedback test', model: 'test', promptTier: 'full', contextWindow: 128000,supportsTools: false, supportsVision: false, ...implementation };@@ -72,7 +95,8 @@ for (const build of ['chrome', 'firefox']) {const tab = nextTab++, agent = setup(Agent);const previousSendMessage = api.tabs.sendMessage;const previousEnable = cdpClient.enableDevDiagnostics, previousEvaluate = cdpClient.evaluate;- let expression = '';+ const previousEvaluateFunction = cdpClient.evaluateFunction;+ let declaration = '', passedArgs = [];try {await agent._claimRunEntry(tab, 'interactive');await agent._beginPageFeedbackRun(tab, 'interactive');@@ -83,22 +107,29 @@ for (const build of ['chrome', 'firefox']) {revision: 7, operationId: message.params.operationId } }: { ready: true };cdpClient.enableDevDiagnostics = async () => {};- cdpClient.evaluate = async (_tab, value) => {- expression = value;- const token = value.match(/__webbrainPageFeedbackAborted: "([^"]+)"/)?.[1];- assert.ok(token, 'The wrapped page expression must have a unique aborted-dispatch result');- return { result: { value: { __webbrainPageFeedbackAborted: token } } };+ cdpClient.evaluateFunction = async (_tab, value, args) => {+ declaration = value; passedArgs = args;+ assert.equal(typeof args[0], 'object');+ assert.equal(typeof args[1], 'string');+ return { result: { value: { __webbrainPageFeedbackAborted: args[1] } } };};const result = await agent._executeDevJavaScript(tab, { code: 'window.__shouldNotRun = true;' });assert.equal(result.dispatched, false);assert.equal(result.noDispatch, true);assert.match(result.error, /page changed during JavaScript preparation/i);- assert.match(expression, /cancelable: true/);- assert.match(expression, /if \(!window\.dispatchEvent\(gate\)\) return/);- assert.ok(expression.indexOf('return (async () =>') > expression.indexOf('dispatchEvent(gate)'));+ assert.match(declaration, /cancelable: true/);+ assert.match(declaration, /if \(!window\.dispatchEvent\(gate\)\) return/);+ assert.ok(declaration.indexOf('return (async () =>') > declaration.indexOf('dispatchEvent(gate)'));+ assert.match(declaration, /window\.__shouldNotRun = true/);+ assert.doesNotMatch(declaration, /execute-js-document|operationId/,+ 'The page guard must travel as a protocol argument instead of source text');+ assert.equal(passedArgs[0].documentToken, 'execute-js-document');+ assert.ok(passedArgs[1]);+ assert.equal(cdpClient.evaluate, previousEvaluate, 'Guarded execution must not use Runtime.evaluate source construction');} finally {cdpClient.enableDevDiagnostics = previousEnable;cdpClient.evaluate = previousEvaluate;+ cdpClient.evaluateFunction = previousEvaluateFunction;api.tabs.sendMessage = previousSendMessage;if (agent._pageFeedbackRuns.has(tab)) agent._finishPageFeedbackRun(tab);agent._releaseRunEntry(tab);diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 7c394c33b..ab73838ea 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1891,6 +1891,35 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: native hover markers attribute hover-triggered page changes to the agent`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const guard = await page.evaluate(() => {+ feedback = [];+ document.getElementById('agent').addEventListener('pointerover', () => {+ document.getElementById('status').textContent = 'Pointer hover changed';+ });+ document.getElementById('agent').addEventListener('mouseover', () => {+ document.getElementById('status').textContent = 'Mouse hover changed';+ });+ deliver('page_monitor_prepare', { operationId: 'native-hover', tool: 'hover', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'native-hover', kind: 'click', navigationCandidate: false, fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ const marker = createNativeActionMarker(guard, 'click', 1);+ await page.locator('#agent').evaluate((element, value) => element.setAttribute('data-webbrain-native-action', value), marker);+ await page.locator('#agent').hover();+ assert.match(await page.locator('#status').textContent(), /hover changed/i);+ await page.waitForTimeout(200);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,+ 'The first trusted hover event must activate and attribute its prepared native operation');+ await page.locator('#human').click();+ await page.waitForTimeout(200);+ assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'),+ 'A subsequent user click remains visible to the monitor');+ } finally { await browser.close(); }+ });+test(`${build}: native uploads target an unfocused file input without hiding user input`, async () => {const { browser, page } = await fixture(engine, build);const session = new BidiSession(), runId = crypto.randomUUID(), token = crypto.randomUUID();