Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(firefox): address BiDi review feedback and CI runtime
Emre Sokulluf147d0c1d5a5
diff --git a/.github/workflows/main.yml b/.github/workflows/main.ymlindex 864152a4b..2b75e2949 100644--- a/.github/workflows/main.yml+++ b/.github/workflows/main.yml@@ -21,7 +21,7 @@ jobs:- uses: actions/setup-node@v4with:- node-version: 20+ node-version: 22- name: Install dependenciesrun: npm cidiff --git a/.github/workflows/minor-release.yml b/.github/workflows/minor-release.ymlindex a37329f64..ca4959a36 100644--- a/.github/workflows/minor-release.yml+++ b/.github/workflows/minor-release.yml@@ -32,7 +32,7 @@ jobs:- uses: actions/setup-node@v4with:- node-version: 20+ node-version: 22- name: Install dependenciesrun: npm cidiff --git a/firefox-companion/README.md b/firefox-companion/README.mdindex 9b717bd6d..00908346c 100644--- a/firefox-companion/README.md+++ b/firefox-companion/README.md@@ -22,6 +22,7 @@ The connection test checks the real BiDi session. It does not submit forms or se## First supported scope- Session connection through Firefox native messaging to loopback BiDi; no arbitrary protocol or script tool exposed to the model.+- Coordinate clicks preserve the validated CSS viewport point and recheck that point against the bound element before dispatch. Pre-dispatch validation failures retain `noDispatch:true`; transport failures and failures after input remain uncertain and unsafe to replay.- Trusted clicks (`click`, `click_ax`), hover, text (`type_text`, `type_ax`, `set_field`), and existing supported `press_keys` keys/repeats. Native selects retain their existing semantic selection path. Drag/drop and dedicated iframe tools retain their existing implementations.- Typing dispatches one character at a time, checking run ownership, deadline and focus before each step. Stop prevents the next step; a single already-dispatched character can still arrive. Newlines use a literal editing command, never Enter; single-line fields reject multiline payloads before clearing.- Exact value readback after typing. `set_field` submits with Enter only after verification. An uncertain dispatch is not safe to repeat automatically.diff --git a/firefox-companion/host.mjs b/firefox-companion/host.mjsindex ef03d5086..6d278bab4 100644--- a/firefox-companion/host.mjs+++ b/firefox-companion/host.mjs@@ -19,7 +19,7 @@ async function dispatch(message) {default: throw new Error('Unknown companion command');}reply({ id, result });- } catch (error) { reply({ id, error: error.message }); }+ } catch (error) { reply({ id, error: error.message, ...(error.dispatchState ? { dispatchState: error.dispatchState } : {}) }); }}process.stdin.on('data', data => {buffer = Buffer.concat([buffer, data]);diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 2037ed133..364a93f3c 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -110,6 +110,15 @@ export class BidiSession {return {};}async perform(id, action, payload) {+ const dispatch = { started: false };+ try { return await this.performAction(id, action, payload, dispatch); }+ catch (error) {+ error.dispatchState = { dispatched: dispatch.started, noDispatch: !dispatch.started,+ outcomeUnknown: dispatch.started, retryable: !dispatch.started };+ throw error;+ }+ }+ async performAction(id, action, payload, dispatch) {if (!['navigate', 'click', 'hover', 'type', 'field', 'key', 'upload'].includes(action)) throw new Error('Unsupported BiDi action');if (payload.modifiers) throw new Error('Key modifiers are not supported by this tool');const run = this.runs.get(id); if (!run) throw new Error('Run stopped or disconnected');@@ -121,25 +130,30 @@ export class BidiSession {if (action === 'navigate') {if (!/^https?:\/\//.test(payload.url)) throw new Error('Invalid navigation URL');run.navigation = true;- try { await this.send('browsingContext.navigate', { context: run.context, url: payload.url, wait: 'interactive' }); }+ try { dispatch.started = true; await this.send('browsingContext.navigate', { context: run.context, url: payload.url, wait: 'interactive' }); }finally { run.navigation = false; }return { success: true, dispatched: true };}const match = await this.locate(payload.token, payload.url, run.context);assertLive();- const check = await this.call(match, `(el, token, action) => {+ const point = action === 'click' && payload.point != null ? payload.point : null;+ if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');+ const check = await this.call(match, `(el, token, action, x, y) => {if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;el.removeAttribute('data-webbrain-bidi');if (action === 'upload') return el.tagName === 'INPUT' && el.type === 'file';const r = el.getBoundingClientRect();- const hit = document.elementFromPoint(r.x+r.width/2, r.y+r.height/2);+ const px = x === null ? r.x+r.width/2 : x;+ const py = y === null ? r.y+r.height/2 : y;+ if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;+ const hit = document.elementFromPoint(px, py);if (!r.width || !r.height || !(hit === el || el.contains(hit))) return false;if (action === 'type' || action === 'field' || action === 'key') {el.focus({preventScroll:true});if (el.getRootNode().activeElement !== el) return false;}return true;- }`, [{ type: 'string', value: payload.token }, { type: 'string', value: action }]);+ }`, [{ type: 'string', value: payload.token }, { type: 'string', value: action }, point ? {type:'number',value:point.x} : {type:'null'}, point ? {type:'number',value:point.y} : {type:'null'}]);if (check.result?.value !== true) throw new Error('Target changed or is covered; no input sent');assertLive();if (action === 'upload') {@@ -151,6 +165,7 @@ export class BidiSession {const path = join(dir, name === '.' || name === '..' ? 'attachment' : name);await writeFile(path, bytes, { mode: 0o600 });assertLive();+ dispatch.started = true;await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });const attached = await this.call(match, '(el, name, size) => el.isConnected && el.files?.length === 1 && el.files[0].name === name && el.files[0].size === size', [{ type: 'string', value: name }, { type: 'number', value: bytes.length }]);if (attached.result?.value !== true) return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'File input changed after attachment; inspect the page before retrying.' };@@ -173,17 +188,18 @@ export class BidiSession {{ type: 'keyDown', value }, { type: 'keyUp', value },...(modifier ? [{ type: 'keyUp', value: modifier }] : []),];- try { await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] }); }+ try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] }); }finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }assertLive();};if (action === 'click' || action === 'hover') {const source = { type: 'pointer', id: 'webbrain-pointer', parameters: { pointerType: 'mouse' }, actions: [- { type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: match.node.sharedId } } },+ point ? { type: 'pointerMove', x: point.x, y: point.y, origin: 'viewport' }+ : { type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: match.node.sharedId } } },...(action === 'click' ? [{ type: 'pointerDown', button: 0 }, { type: 'pointerUp', button: 0 }] : []),] };assertLive();- try { await this.send('input.performActions', { context: match.context, actions: [source] }); }+ try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }assertLive();} else if (typing) {@@ -203,6 +219,7 @@ export class BidiSession {await assertFocus();// Enter is a submit shortcut on many editors, even with Shift. Insert a literal// newline through the editing command instead; never synthesize a submit key.+ dispatch.started = true;const inserted = await this.call(match, `(el, text) => {if (!el.isConnected || el.getRootNode().activeElement !== el) return false;return document.execCommand('insertText', false, text);diff --git a/src/firefox/src/bidi/bind.js b/src/firefox/src/bidi/bind.jsnew file mode 100644index 000000000..a1d2c1970--- /dev/null+++ b/src/firefox/src/bidi/bind.js@@ -0,0 +1,14 @@+// Packaged code avoids constructing executable source from binding data.+(() => {+ const bytes = crypto.getRandomValues(new Uint8Array(16));+ bytes[6] = (bytes[6] & 15) | 64;+ bytes[8] = (bytes[8] & 63) | 128;+ const hex = Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('');+ const token = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;+ const el = document.documentElement;+ el.setAttribute('data-webbrain-bidi', token);+ setTimeout(() => {+ if (el.getAttribute('data-webbrain-bidi') === token) el.removeAttribute('data-webbrain-bidi');+ }, 10000);+ return { token, url: location.href };+})();diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex cf7655084..516c8d984 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -8,7 +8,13 @@ export class FirefoxBidiClient {this.port.onMessage.addListener(message => {const entry = this.pending.get(message.id); if (!entry) return;this.pending.delete(message.id);- message.error ? entry.reject(new Error(message.error)) : entry.resolve(message.result);+ if (message.error) {+ const error = new Error(message.error);+ if (message.dispatchState?.dispatched === false && message.dispatchState?.noDispatch === true) {+ error.dispatchState = { dispatched: false, noDispatch: true, outcomeUnknown: false, retryable: true };+ }+ entry.reject(error);+ } else entry.resolve(message.result);});const port = this.port;this.port.onDisconnect.addListener(() => {@@ -54,8 +60,8 @@ export class FirefoxBidiClient {// Internal/new tabs cannot accept content scripts. Keep ownership but defer// document binding so the normal authorized navigation can leave that page.if (!/^https?:\/\//.test(tab.url || '')) return;- const token = crypto.randomUUID();- const [url] = await this.api.tabs.executeScript(tabId, { frameId: 0, code: `(() => { const el = document.documentElement; const token = ${JSON.stringify(token)}; el.setAttribute('data-webbrain-bidi', token); setTimeout(() => { if (el.getAttribute('data-webbrain-bidi') === token) el.removeAttribute('data-webbrain-bidi'); }, 10000); return location.href; })()` });+ const [binding] = await this.api.tabs.executeScript(tabId, { frameId: 0, file: '/src/bidi/bind.js' });+ const {token, url} = binding || {};if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) throw new Error('Run stopped');await this.request('openRun', { runId: owner.runId, token, url });if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) {@@ -103,10 +109,10 @@ export class FirefoxBidiClient {if (message.actionDeadlineAt && Date.now() >= message.actionDeadlineAt) return { success: false, dispatched: false, noDispatch: true, deadlineExpired: true };const metadata = { fieldMeta: prepared.fieldMeta, ...(message.params?.ref_id ? { ref_id: message.params.ref_id } : {}) };try {- return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };+ return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, point: prepared.point || null, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };} catch (error) {// A transport failure can occur after trusted input was delivered.- return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: error.message };+ return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, ...(error.dispatchState || {}), error: error.message };}}}diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 2ae2e96bd..ca34126ac 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2097,7 +2097,12 @@if (actionDeadlineExpired()) return deadlineFailure();const clickedRect = rememberInteractionPoint(el, 'click');if (actionDeadlineExpired()) return deadlineFailure();- if (params._bidiPrepare) return { ...prepareBidiTarget(el, params._bidiPrepare), _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };+ if (params._bidiPrepare) {+ const coordinateClick = !params.text && !params.selector && params.index == null && Number.isFinite(params.x) && Number.isFinite(params.y);+ return { ...prepareBidiTarget(el, params._bidiPrepare),+ ...(coordinateClick ? { point: { x: Math.round(params.x), y: Math.round(params.y) } } : {}),+ _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };+ }dispatched = true;const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());if (filePickerGuard.blocked) {diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjsindex ae288bdf5..74bfd51db 100644--- a/test/firefox-bidi-e2e.mjs+++ b/test/firefox-bidi-e2e.mjs@@ -65,6 +65,17 @@ try {return result.result;};const tabId = (await extensionEval(`browser.tabs.query({}).then(tabs => tabs.find(tab => tab.url === ${JSON.stringify(url)}).id)`)).value;+ await evaluate(`document.body.insertAdjacentHTML('beforeend','<canvas id="point-canvas" style="position:fixed;left:10px;top:200px;width:300px;height:60px"></canvas>'); document.querySelector('#point-canvas').onclick=e=>window.clickedPoint={x:e.clientX,y:e.clientY,trusted:e.isTrusted};`);+ const pointToken = crypto.randomUUID();+ const pointResponse = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content',action:'click',params:{x:22,y:210,coordinate_space:'css',_bidiPrepare:${JSON.stringify(pointToken)}}}).then(value=>JSON.stringify(value))`);+ const pointPrepared = JSON.parse(pointResponse.value);+ assert.deepEqual(pointPrepared.point,{x:22,y:210});+ await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});+ assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});+ const obscured = await mark('#point-canvas');+ await evaluate(`document.body.insertAdjacentHTML('beforeend','<div id="point-cover" style="position:fixed;left:20px;top:205px;width:10px;height:10px;background:red;z-index:99999"></div>')`);+ await assert.rejects(session.perform(runId,'click',{...obscured,point:{x:22,y:210}}), error=>error.dispatchState.noDispatch===true && /covered/.test(error.message));+ await evaluate(`document.querySelector('#point-canvas').remove();document.querySelector('#point-cover').remove()`);const token = crypto.randomUUID();const prepared = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content', action:'type', params:{selector:'#text', text:'Through extension', clear:true, _bidiPrepare:${JSON.stringify(token)}}}).then(value => JSON.stringify(value))`);assert.equal(JSON.parse(prepared.value).bidiPrepared, true, 'packaged content script prepares a target');diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex e330a5c36..121c65042 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -88,6 +88,13 @@ test('installer and native launcher preserve Firefox message framing and commandassert.equal(result.status, 0, result.stderr?.toString());assert.equal(result.stdout.readUInt32LE(0), result.stdout.length - 4);assert.deepEqual(JSON.parse(result.stdout.subarray(4)), { id: 12, error: 'Unknown companion command' });+ const rejected = Buffer.from(JSON.stringify({id:13,command:'perform',runId:id(),action:'click',payload:{}}));+ const rejectedHeader = Buffer.alloc(4); rejectedHeader.writeUInt32LE(rejected.length);+ const rejectedResult = spawnSync(manifest.path, [], {input:Buffer.concat([rejectedHeader,rejected]),timeout:5000});+ assert.equal(rejectedResult.status,0,rejectedResult.stderr?.toString());+ const failure = JSON.parse(rejectedResult.stdout.subarray(4));+ assert.equal(failure.id,13);+ assert.deepEqual(failure.dispatchState,{dispatched:false,noDispatch:true,outcomeUnknown:false,retryable:true});} finally { await rm(directory, { recursive: true, force: true }); }});@@ -105,7 +112,7 @@ test('blank-tab runs defer binding and retain ownership through navigation', asytabs: {get: async () => ({url}),update: async (_id, value) => { url = value.url; },- executeScript: async () => { injections++; return [url]; },+ executeScript: async (_tab, options) => { injections++; assert.equal(options.file, '/src/bidi/bind.js'); assert.equal(options.code, undefined); return [{token:id(), url}]; },},});client.connect = async () => ({});@@ -170,3 +177,59 @@ test('literal newline insertion never dispatches an Enter key', async () => {assert.ok(scripts.some(fn=>fn.includes("execCommand('insertText'")));assert.deepEqual(dispatched.flatMap(p=>p.actions[0].actions.map(a=>a.value)), ['a','a','b','b']);});++test('validation failures preserve no-dispatch status across the native client', async () => {+ const session = new BidiSession(); const runId = id();+ session.runs.set(runId, {context:'tab'});+ session.locate = async () => { throw new Error('stale target'); };+ let receive;+ const client = new FirefoxBidiClient({+ runtime: { connectNative: () => ({+ onMessage: {addListener: listener => {receive=listener;}}, onDisconnect: {addListener(){}},+ postMessage: message => {+ session.perform(message.runId, message.action, message.payload).then(+ result => receive({id:message.id,result}),+ error => receive({id:message.id,error:error.message,dispatchState:error.dispatchState}),+ );+ },+ }) },+ tabs: {sendMessage: async () => ({bidiPrepared:true,url:'https://example.com/'})},+ });+ client.runs.set(1,{runId,bound:true});+ const result = await client.sendContent(1,{action:'click',params:{}});+ assert.equal(result.dispatched,false);+ assert.equal(result.noDispatch,true);+ assert.equal(result.outcomeUnknown,false);+ assert.equal(result.retryable,true);+});+test('post-dispatch failure and transport uncertainty are never safe retries', async () => {+ const session = new BidiSession(); const runId = id();+ session.runs.set(runId, {context:'tab'});+ session.locate = async () => ({context:'tab',node:{sharedId:'el'}});+ session.call = async () => ({result:{value:true}});+ session.send = async method => { if(method==='input.performActions')throw new Error('connection lost'); return {}; };+ await assert.rejects(session.perform(runId,'click',{}), error => error.dispatchState.dispatched === true && error.dispatchState.retryable === false);+ const client = new FirefoxBidiClient({tabs:{sendMessage:async()=>({bidiPrepared:true,url:'https://example.com/'})}});+ client.runs.set(1,{runId,bound:true}); client.request=async()=>{throw new Error('timeout')};+ const result=await client.sendContent(1,{action:'click',params:{}});+ assert.equal(result.dispatched,true);+ assert.equal(result.outcomeUnknown,true);+ assert.equal(result.retryable,false);+});+test('coordinate clicks retain their validated viewport point', async () => {+ const session = new BidiSession(); const runId = id(); const sent=[];+ session.runs.set(runId,{context:'tab'});+ session.locate=async()=>({context:'tab',node:{sharedId:'canvas'}});+ session.call=async(_match,fn,args)=>{assert.equal(args[2].value,17);assert.equal(args[3].value,23);return {result:{value:true}}};+ session.send=async(method,params)=>{if(method==='input.performActions')sent.push(params);return {}};+ await session.perform(runId,'click',{point:{x:17,y:23}});+ assert.deepEqual(sent[0].actions[0].actions[0],{type:'pointerMove',origin:'viewport',x:17,y:23});+});+test('every release workflow running npm test uses the companion runtime', async () => {+ const {readFile}=await import('node:fs/promises');+ for(const workflow of ['main','minor-release']) {+ const source=await readFile(`.github/workflows/${workflow}.yml`,'utf8');+ assert.match(source,/node-version: 22/);+ assert.match(source,/run: npm test/);+ }+});