Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(firefox): address BiDi review feedback and CI runtime

Emre Sokulluf147d0c1d5a5
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 864152a4b..2b75e2949 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -21,7 +21,7 @@ jobs:
- uses: actions/setup-node@v4
with:
- node-version: 20
+ node-version: 22
- name: Install dependencies
run: npm ci
diff --git a/.github/workflows/minor-release.yml b/.github/workflows/minor-release.yml
index a37329f64..ca4959a36 100644
--- a/.github/workflows/minor-release.yml
+++ b/.github/workflows/minor-release.yml
@@ -32,7 +32,7 @@ jobs:
- uses: actions/setup-node@v4
with:
- node-version: 20
+ node-version: 22
- name: Install dependencies
run: npm ci
diff --git a/firefox-companion/README.md b/firefox-companion/README.md
index 9b717bd6d..00908346c 100644
--- a/firefox-companion/README.md
+++ b/firefox-companion/README.md
@@ -22,6 +22,7 @@ The connection test checks the real BiDi session. It does not submit forms or se
## First supported scope
- Session connection through Firefox native messaging to loopback BiDi; no arbitrary protocol or script tool exposed to the model.
+- Coordinate clicks preserve the validated CSS viewport point and recheck that point against the bound element before dispatch. Pre-dispatch validation failures retain `noDispatch:true`; transport failures and failures after input remain uncertain and unsafe to replay.
- Trusted clicks (`click`, `click_ax`), hover, text (`type_text`, `type_ax`, `set_field`), and existing supported `press_keys` keys/repeats. Native selects retain their existing semantic selection path. Drag/drop and dedicated iframe tools retain their existing implementations.
- Typing dispatches one character at a time, checking run ownership, deadline and focus before each step. Stop prevents the next step; a single already-dispatched character can still arrive. Newlines use a literal editing command, never Enter; single-line fields reject multiline payloads before clearing.
- Exact value readback after typing. `set_field` submits with Enter only after verification. An uncertain dispatch is not safe to repeat automatically.
diff --git a/firefox-companion/host.mjs b/firefox-companion/host.mjs
index ef03d5086..6d278bab4 100644
--- a/firefox-companion/host.mjs
+++ b/firefox-companion/host.mjs
@@ -19,7 +19,7 @@ async function dispatch(message) {
default: throw new Error('Unknown companion command');
}
reply({ id, result });
- } catch (error) { reply({ id, error: error.message }); }
+ } catch (error) { reply({ id, error: error.message, ...(error.dispatchState ? { dispatchState: error.dispatchState } : {}) }); }
}
process.stdin.on('data', data => {
buffer = Buffer.concat([buffer, data]);
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 2037ed133..364a93f3c 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -110,6 +110,15 @@ export class BidiSession {
return {};
}
async perform(id, action, payload) {
+ const dispatch = { started: false };
+ try { return await this.performAction(id, action, payload, dispatch); }
+ catch (error) {
+ error.dispatchState = { dispatched: dispatch.started, noDispatch: !dispatch.started,
+ outcomeUnknown: dispatch.started, retryable: !dispatch.started };
+ throw error;
+ }
+ }
+ async performAction(id, action, payload, dispatch) {
if (!['navigate', 'click', 'hover', 'type', 'field', 'key', 'upload'].includes(action)) throw new Error('Unsupported BiDi action');
if (payload.modifiers) throw new Error('Key modifiers are not supported by this tool');
const run = this.runs.get(id); if (!run) throw new Error('Run stopped or disconnected');
@@ -121,25 +130,30 @@ export class BidiSession {
if (action === 'navigate') {
if (!/^https?:\/\//.test(payload.url)) throw new Error('Invalid navigation URL');
run.navigation = true;
- try { await this.send('browsingContext.navigate', { context: run.context, url: payload.url, wait: 'interactive' }); }
+ try { dispatch.started = true; await this.send('browsingContext.navigate', { context: run.context, url: payload.url, wait: 'interactive' }); }
finally { run.navigation = false; }
return { success: true, dispatched: true };
}
const match = await this.locate(payload.token, payload.url, run.context);
assertLive();
- const check = await this.call(match, `(el, token, action) => {
+ const point = action === 'click' && payload.point != null ? payload.point : null;
+ if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');
+ const check = await this.call(match, `(el, token, action, x, y) => {
if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;
el.removeAttribute('data-webbrain-bidi');
if (action === 'upload') return el.tagName === 'INPUT' && el.type === 'file';
const r = el.getBoundingClientRect();
- const hit = document.elementFromPoint(r.x+r.width/2, r.y+r.height/2);
+ const px = x === null ? r.x+r.width/2 : x;
+ const py = y === null ? r.y+r.height/2 : y;
+ if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;
+ const hit = document.elementFromPoint(px, py);
if (!r.width || !r.height || !(hit === el || el.contains(hit))) return false;
if (action === 'type' || action === 'field' || action === 'key') {
el.focus({preventScroll:true});
if (el.getRootNode().activeElement !== el) return false;
}
return true;
- }`, [{ type: 'string', value: payload.token }, { type: 'string', value: action }]);
+ }`, [{ type: 'string', value: payload.token }, { type: 'string', value: action }, point ? {type:'number',value:point.x} : {type:'null'}, point ? {type:'number',value:point.y} : {type:'null'}]);
if (check.result?.value !== true) throw new Error('Target changed or is covered; no input sent');
assertLive();
if (action === 'upload') {
@@ -151,6 +165,7 @@ export class BidiSession {
const path = join(dir, name === '.' || name === '..' ? 'attachment' : name);
await writeFile(path, bytes, { mode: 0o600 });
assertLive();
+ dispatch.started = true;
await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });
const attached = await this.call(match, '(el, name, size) => el.isConnected && el.files?.length === 1 && el.files[0].name === name && el.files[0].size === size', [{ type: 'string', value: name }, { type: 'number', value: bytes.length }]);
if (attached.result?.value !== true) return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'File input changed after attachment; inspect the page before retrying.' };
@@ -173,17 +188,18 @@ export class BidiSession {
{ type: 'keyDown', value }, { type: 'keyUp', value },
...(modifier ? [{ type: 'keyUp', value: modifier }] : []),
];
- try { await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] }); }
+ try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] }); }
finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
assertLive();
};
if (action === 'click' || action === 'hover') {
const source = { type: 'pointer', id: 'webbrain-pointer', parameters: { pointerType: 'mouse' }, actions: [
- { type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: match.node.sharedId } } },
+ point ? { type: 'pointerMove', x: point.x, y: point.y, origin: 'viewport' }
+ : { type: 'pointerMove', x: 0, y: 0, origin: { type: 'element', element: { sharedId: match.node.sharedId } } },
...(action === 'click' ? [{ type: 'pointerDown', button: 0 }, { type: 'pointerUp', button: 0 }] : []),
] };
assertLive();
- try { await this.send('input.performActions', { context: match.context, actions: [source] }); }
+ try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }
finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
assertLive();
} else if (typing) {
@@ -203,6 +219,7 @@ export class BidiSession {
await assertFocus();
// Enter is a submit shortcut on many editors, even with Shift. Insert a literal
// newline through the editing command instead; never synthesize a submit key.
+ dispatch.started = true;
const inserted = await this.call(match, `(el, text) => {
if (!el.isConnected || el.getRootNode().activeElement !== el) return false;
return document.execCommand('insertText', false, text);
diff --git a/src/firefox/src/bidi/bind.js b/src/firefox/src/bidi/bind.js
new file mode 100644
index 000000000..a1d2c1970
--- /dev/null
+++ b/src/firefox/src/bidi/bind.js
@@ -0,0 +1,14 @@
+// Packaged code avoids constructing executable source from binding data.
+(() => {
+ const bytes = crypto.getRandomValues(new Uint8Array(16));
+ bytes[6] = (bytes[6] & 15) | 64;
+ bytes[8] = (bytes[8] & 63) | 128;
+ const hex = Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('');
+ const token = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+ const el = document.documentElement;
+ el.setAttribute('data-webbrain-bidi', token);
+ setTimeout(() => {
+ if (el.getAttribute('data-webbrain-bidi') === token) el.removeAttribute('data-webbrain-bidi');
+ }, 10000);
+ return { token, url: location.href };
+})();
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index cf7655084..516c8d984 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -8,7 +8,13 @@ export class FirefoxBidiClient {
this.port.onMessage.addListener(message => {
const entry = this.pending.get(message.id); if (!entry) return;
this.pending.delete(message.id);
- message.error ? entry.reject(new Error(message.error)) : entry.resolve(message.result);
+ if (message.error) {
+ const error = new Error(message.error);
+ if (message.dispatchState?.dispatched === false && message.dispatchState?.noDispatch === true) {
+ error.dispatchState = { dispatched: false, noDispatch: true, outcomeUnknown: false, retryable: true };
+ }
+ entry.reject(error);
+ } else entry.resolve(message.result);
});
const port = this.port;
this.port.onDisconnect.addListener(() => {
@@ -54,8 +60,8 @@ export class FirefoxBidiClient {
// Internal/new tabs cannot accept content scripts. Keep ownership but defer
// document binding so the normal authorized navigation can leave that page.
if (!/^https?:\/\//.test(tab.url || '')) return;
- const token = crypto.randomUUID();
- const [url] = await this.api.tabs.executeScript(tabId, { frameId: 0, code: `(() => { const el = document.documentElement; const token = ${JSON.stringify(token)}; el.setAttribute('data-webbrain-bidi', token); setTimeout(() => { if (el.getAttribute('data-webbrain-bidi') === token) el.removeAttribute('data-webbrain-bidi'); }, 10000); return location.href; })()` });
+ const [binding] = await this.api.tabs.executeScript(tabId, { frameId: 0, file: '/src/bidi/bind.js' });
+ const {token, url} = binding || {};
if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) throw new Error('Run stopped');
await this.request('openRun', { runId: owner.runId, token, url });
if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) {
@@ -103,10 +109,10 @@ export class FirefoxBidiClient {
if (message.actionDeadlineAt && Date.now() >= message.actionDeadlineAt) return { success: false, dispatched: false, noDispatch: true, deadlineExpired: true };
const metadata = { fieldMeta: prepared.fieldMeta, ...(message.params?.ref_id ? { ref_id: message.params.ref_id } : {}) };
try {
- return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };
+ return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, point: prepared.point || null, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };
} catch (error) {
// A transport failure can occur after trusted input was delivered.
- return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: error.message };
+ return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, ...(error.dispatchState || {}), error: error.message };
}
}
}
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 2ae2e96bd..ca34126ac 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -2097,7 +2097,12 @@
if (actionDeadlineExpired()) return deadlineFailure();
const clickedRect = rememberInteractionPoint(el, 'click');
if (actionDeadlineExpired()) return deadlineFailure();
- if (params._bidiPrepare) return { ...prepareBidiTarget(el, params._bidiPrepare), _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };
+ if (params._bidiPrepare) {
+ const coordinateClick = !params.text && !params.selector && params.index == null && Number.isFinite(params.x) && Number.isFinite(params.y);
+ return { ...prepareBidiTarget(el, params._bidiPrepare),
+ ...(coordinateClick ? { point: { x: Math.round(params.x), y: Math.round(params.y) } } : {}),
+ _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };
+ }
dispatched = true;
const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
if (filePickerGuard.blocked) {
diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjs
index ae288bdf5..74bfd51db 100644
--- a/test/firefox-bidi-e2e.mjs
+++ b/test/firefox-bidi-e2e.mjs
@@ -65,6 +65,17 @@ try {
return result.result;
};
const tabId = (await extensionEval(`browser.tabs.query({}).then(tabs => tabs.find(tab => tab.url === ${JSON.stringify(url)}).id)`)).value;
+ await evaluate(`document.body.insertAdjacentHTML('beforeend','<canvas id="point-canvas" style="position:fixed;left:10px;top:200px;width:300px;height:60px"></canvas>'); document.querySelector('#point-canvas').onclick=e=>window.clickedPoint={x:e.clientX,y:e.clientY,trusted:e.isTrusted};`);
+ const pointToken = crypto.randomUUID();
+ const pointResponse = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content',action:'click',params:{x:22,y:210,coordinate_space:'css',_bidiPrepare:${JSON.stringify(pointToken)}}}).then(value=>JSON.stringify(value))`);
+ const pointPrepared = JSON.parse(pointResponse.value);
+ assert.deepEqual(pointPrepared.point,{x:22,y:210});
+ await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});
+ assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});
+ const obscured = await mark('#point-canvas');
+ await evaluate(`document.body.insertAdjacentHTML('beforeend','<div id="point-cover" style="position:fixed;left:20px;top:205px;width:10px;height:10px;background:red;z-index:99999"></div>')`);
+ await assert.rejects(session.perform(runId,'click',{...obscured,point:{x:22,y:210}}), error=>error.dispatchState.noDispatch===true && /covered/.test(error.message));
+ await evaluate(`document.querySelector('#point-canvas').remove();document.querySelector('#point-cover').remove()`);
const token = crypto.randomUUID();
const prepared = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content', action:'type', params:{selector:'#text', text:'Through extension', clear:true, _bidiPrepare:${JSON.stringify(token)}}}).then(value => JSON.stringify(value))`);
assert.equal(JSON.parse(prepared.value).bidiPrepared, true, 'packaged content script prepares a target');
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index e330a5c36..121c65042 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -88,6 +88,13 @@ test('installer and native launcher preserve Firefox message framing and command
assert.equal(result.status, 0, result.stderr?.toString());
assert.equal(result.stdout.readUInt32LE(0), result.stdout.length - 4);
assert.deepEqual(JSON.parse(result.stdout.subarray(4)), { id: 12, error: 'Unknown companion command' });
+ const rejected = Buffer.from(JSON.stringify({id:13,command:'perform',runId:id(),action:'click',payload:{}}));
+ const rejectedHeader = Buffer.alloc(4); rejectedHeader.writeUInt32LE(rejected.length);
+ const rejectedResult = spawnSync(manifest.path, [], {input:Buffer.concat([rejectedHeader,rejected]),timeout:5000});
+ assert.equal(rejectedResult.status,0,rejectedResult.stderr?.toString());
+ const failure = JSON.parse(rejectedResult.stdout.subarray(4));
+ assert.equal(failure.id,13);
+ assert.deepEqual(failure.dispatchState,{dispatched:false,noDispatch:true,outcomeUnknown:false,retryable:true});
} finally { await rm(directory, { recursive: true, force: true }); }
});
@@ -105,7 +112,7 @@ test('blank-tab runs defer binding and retain ownership through navigation', asy
tabs: {
get: async () => ({url}),
update: async (_id, value) => { url = value.url; },
- executeScript: async () => { injections++; return [url]; },
+ executeScript: async (_tab, options) => { injections++; assert.equal(options.file, '/src/bidi/bind.js'); assert.equal(options.code, undefined); return [{token:id(), url}]; },
},
});
client.connect = async () => ({});
@@ -170,3 +177,59 @@ test('literal newline insertion never dispatches an Enter key', async () => {
assert.ok(scripts.some(fn=>fn.includes("execCommand('insertText'")));
assert.deepEqual(dispatched.flatMap(p=>p.actions[0].actions.map(a=>a.value)), ['a','a','b','b']);
});
+
+test('validation failures preserve no-dispatch status across the native client', async () => {
+ const session = new BidiSession(); const runId = id();
+ session.runs.set(runId, {context:'tab'});
+ session.locate = async () => { throw new Error('stale target'); };
+ let receive;
+ const client = new FirefoxBidiClient({
+ runtime: { connectNative: () => ({
+ onMessage: {addListener: listener => {receive=listener;}}, onDisconnect: {addListener(){}},
+ postMessage: message => {
+ session.perform(message.runId, message.action, message.payload).then(
+ result => receive({id:message.id,result}),
+ error => receive({id:message.id,error:error.message,dispatchState:error.dispatchState}),
+ );
+ },
+ }) },
+ tabs: {sendMessage: async () => ({bidiPrepared:true,url:'https://example.com/'})},
+ });
+ client.runs.set(1,{runId,bound:true});
+ const result = await client.sendContent(1,{action:'click',params:{}});
+ assert.equal(result.dispatched,false);
+ assert.equal(result.noDispatch,true);
+ assert.equal(result.outcomeUnknown,false);
+ assert.equal(result.retryable,true);
+});
+test('post-dispatch failure and transport uncertainty are never safe retries', async () => {
+ const session = new BidiSession(); const runId = id();
+ session.runs.set(runId, {context:'tab'});
+ session.locate = async () => ({context:'tab',node:{sharedId:'el'}});
+ session.call = async () => ({result:{value:true}});
+ session.send = async method => { if(method==='input.performActions')throw new Error('connection lost'); return {}; };
+ await assert.rejects(session.perform(runId,'click',{}), error => error.dispatchState.dispatched === true && error.dispatchState.retryable === false);
+ const client = new FirefoxBidiClient({tabs:{sendMessage:async()=>({bidiPrepared:true,url:'https://example.com/'})}});
+ client.runs.set(1,{runId,bound:true}); client.request=async()=>{throw new Error('timeout')};
+ const result=await client.sendContent(1,{action:'click',params:{}});
+ assert.equal(result.dispatched,true);
+ assert.equal(result.outcomeUnknown,true);
+ assert.equal(result.retryable,false);
+});
+test('coordinate clicks retain their validated viewport point', async () => {
+ const session = new BidiSession(); const runId = id(); const sent=[];
+ session.runs.set(runId,{context:'tab'});
+ session.locate=async()=>({context:'tab',node:{sharedId:'canvas'}});
+ session.call=async(_match,fn,args)=>{assert.equal(args[2].value,17);assert.equal(args[3].value,23);return {result:{value:true}}};
+ session.send=async(method,params)=>{if(method==='input.performActions')sent.push(params);return {}};
+ await session.perform(runId,'click',{point:{x:17,y:23}});
+ assert.deepEqual(sent[0].actions[0].actions[0],{type:'pointerMove',origin:'viewport',x:17,y:23});
+});
+test('every release workflow running npm test uses the companion runtime', async () => {
+ const {readFile}=await import('node:fs/promises');
+ for(const workflow of ['main','minor-release']) {
+ const source=await readFile(`.github/workflows/${workflow}.yml`,'utf8');
+ assert.match(source,/node-version: 22/);
+ assert.match(source,/run: npm test/);
+ }
+});