Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Prepare Firefox BiDi target frame monitor
Barack Sokullu6feda9b5d056
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex 9038c3715..127941793 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -158,6 +158,7 @@ export class FirefoxBidiClient {: ['type', 'field', 'key', 'upload'].includes(action) ? 'input' : 'click',...(payload.point ? { x: payload.point.x, y: payload.point.y } : {}),selector: payload.selector, ref_id: payload.ref_id, frameId: payload.frameId || 0,+ tool: action, prepareMonitor: true,navigationCandidate: ['click', 'checked'].includes(action),fenceOnly: true,url: action === 'navigate' ? payload.url : undefined,diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex bfbddbd89..887aba89b 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';import vm from 'node:vm';import { FirefoxBidiClient } from '../src/firefox/src/bidi/client.js';import { BidiSession } from '../firefox-companion/session.mjs';-import { installPageFeedback, pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';+import { installPageFeedback, pageFeedbackMethods, validateNativePageDispatch } from '../src/firefox/src/agent/page-feedback.js';const id = () => crypto.randomUUID();@@ -12,17 +12,19 @@ test('uploads register input dispatches with the extension monitor', async () =>const guard = { documentToken: 'file-document', revision: 0, operationId: 'upload' };const registrations = [];const client = new FirefoxBidiClient({ tabs: { sendMessage: async (_tab, message) => {- registrations.push(message.params); return { guard };+ registrations.push(message);+ return message.action === 'page_monitor_prepare' ? { ready: true } : { ready: true, guard };} } });client.runs.set(tab, { runId: id(), bound: true });client.request = async (_command, args) => { assert.deepEqual(args.payload.pageFeedbackGuard, guard); return { success: true }; };await owner._beginPageFeedbackRun(tab, 'interactive');try {await client.perform(tab, 'upload', { selector: '#file' });- assert.equal(registrations.length, 1);- assert.equal(registrations[0].kind, 'input');- assert.equal(registrations[0].navigationCandidate, false);- assert.equal(registrations[0].fenceOnly, true);+ assert.deepEqual(registrations.map(message => message.action), ['page_monitor_prepare', 'page_monitor_dispatch']);+ assert.equal(registrations[0].params.tool, 'upload');+ assert.equal(registrations[1].params.kind, 'input');+ assert.equal(registrations[1].params.navigationCandidate, false);+ assert.equal(registrations[1].params.fenceOnly, true);} finally { owner._finishPageFeedbackRun(tab); }});@@ -154,6 +156,73 @@ test('native dispatch validation returns through the extension-owned frame monitelse globalThis.browser = originalBrowser;}});+test('BiDi prepares the target frame before validating native dispatch', async () => {+ const originalBrowser = globalThis.browser;+ const tabId = 957, runId = id(), deliveries = [], prepared = new Map();+ const api = {+ tabs: {+ get: async () => ({ url: 'https://example.test/' }),+ sendMessage: async (tab, message, options) => {+ const frameId = options?.frameId || 0;+ const params = message.params || {};+ deliveries.push({ tab, frameId, action: message.action, params });+ const key = `${frameId}:${params.operationId}`;+ if (message.action === 'page_monitor_prepare') {+ prepared.set(key, params);+ return { ready: true };+ }+ if (message.action === 'page_monitor_dispatch') return { ready: true, guard: {+ documentToken: frameId === 9 ? 'document-9' : 'document-0', revision: 4,+ operationId: params.operationId, runToken: params.runToken,+ } };+ if (message.action === 'page_monitor_validate') return { ready: prepared.has(key) };+ return { ready: true };+ },+ },+ webNavigation: { getAllFrames: async () => [+ { frameId: 0, documentId: 'doc-0', url: 'https://example.test/' },+ { frameId: 9, documentId: 'doc-9', url: 'https://frame.test/' },+ ] },+ };+ class TestAgent {+ static STATE_CHANGE_TOOLS = new Set(['click']);+ isRunning() { return true; }+ _checkAbort() { return false; }+ _hasPendingPageFeedback() { return false; }+ async executeTool(tab, action, args) {+ this.pageMonitorState({ tab: { id: tab }, frameId: 9, documentId: 'doc-9', url: 'https://frame.test/' }, 'document-9');+ return client.perform(tab, action, { ...args, frameId: 9 });+ }+ }+ installPageFeedback(TestAgent);+ const agent = new TestAgent();+ const client = new FirefoxBidiClient(api);+ client.runs.set(tabId, { runId, bound: true });+ globalThis.browser = api;+ let validatedGuard;+ client.request = async (_command, args) => {+ validatedGuard = args.payload.pageFeedbackGuard;+ assert.ok(validatedGuard, 'The child-frame monitor must return a dispatch guard');+ assert.equal(await validateNativePageDispatch(api, tabId, validatedGuard, { kind: 'input' }), true,+ 'The child-frame monitor must validate the prepared operation');+ return { success: true };+ };+ try {+ await agent._beginPageFeedbackRun(tabId, 'interactive');+ await agent.executeTool(tabId, 'click', { selector: '#target' });+ const prepare = deliveries.find(item => item.action === 'page_monitor_prepare' && item.frameId === 9);+ const dispatch = deliveries.find(item => item.action === 'page_monitor_dispatch' && item.frameId === 9);+ assert.ok(prepare, 'The resolved target frame must receive a preparation message');+ assert.ok(dispatch, 'The resolved target frame must receive the dispatch fence');+ assert.equal(prepare.params.operationId, dispatch.params.operationId);+ assert.equal(prepare.params.tool, 'click');+ assert.ok(deliveries.indexOf(prepare) < deliveries.indexOf(dispatch), 'Preparation must precede dispatch');+ } finally {+ agent._finishPageFeedbackRun(tabId);+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});test('dialogs are scoped to a live run and navigation permission is single use', () => {const session = new BidiSession(); const replies = [];session.send = async (method, params) => { replies.push({ method, ...params }); };