Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: verify Firefox BiDi trusted checkable clicks

Emre Sokullua4881b12f658
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 364a93f3c..a1ed9b363 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -173,6 +173,10 @@ export class BidiSession {
}
const typing = action === 'type' || action === 'field';
const clear = action === 'field' ? payload.clear !== false : payload.clear === true;
+ const checkable = action === 'click' && payload.checkable;
+ if (checkable && (!['checkbox', 'radio'].includes(checkable.inputType) || typeof checkable.desiredChecked !== 'boolean')) {
+ throw new Error('Invalid checkable target metadata');
+ }
const before = typing ? await this.call(match, '(el) => el.isContentEditable ? el.innerText : el.value') : null;
const assertFocus = async () => {
assertLive();
@@ -202,6 +206,35 @@ export class BidiSession {
try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }
finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
assertLive();
+ if (checkable) {
+ await new Promise(resolve => setTimeout(resolve, 80));
+ assertLive();
+ const checked = await this.call(match, '(el) => el.isConnected && (el.type === "checkbox" || el.type === "radio") ? !!el.checked : null');
+ const checkedAfter = checked.result?.value;
+ if (typeof checkedAfter !== 'boolean') {
+ return { success: false, dispatched: true, outcomeUnknown: true, retryable: false,
+ error: 'Checkable target changed after trusted click; inspect the page before retrying.' };
+ }
+ const stateMatchesDesired = checkedAfter === checkable.desiredChecked;
+ return {
+ success: stateMatchesDesired,
+ dispatched: true,
+ verified: stateMatchesDesired,
+ checkedBefore: !!checkable.checkedBefore,
+ checkedAfter,
+ checkedChanged: !!checkable.checkedBefore !== checkedAfter,
+ desiredChecked: checkable.desiredChecked,
+ checkboxIdentity: checkable.checkboxIdentity,
+ checkboxState: { identity: checkable.checkboxIdentity, desiredChecked: checkable.desiredChecked, actualChecked: checkedAfter },
+ ...(stateMatchesDesired ? { observedEffects: ['checked_state'] } : {
+ noProgress: true,
+ error: checkable.inputType === 'checkbox'
+ ? `Checkbox remained ${checkedAfter ? 'checked' : 'unchecked'} after trusted click.`
+ : 'Radio remained unselected after trusted click.',
+ }),
+ method: 'firefox-bidi',
+ };
+ }
} else if (typing) {
if (typeof payload.text !== 'string' || payload.text.length > 10000) throw new Error('Trusted text input is limited to 10000 characters per action');
if (/[\r\n]/.test(payload.text)) {
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index 516c8d984..02f077ddf 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -109,7 +109,14 @@ export class FirefoxBidiClient {
if (message.actionDeadlineAt && Date.now() >= message.actionDeadlineAt) return { success: false, dispatched: false, noDispatch: true, deadlineExpired: true };
const metadata = { fieldMeta: prepared.fieldMeta, ...(message.params?.ref_id ? { ref_id: message.params.ref_id } : {}) };
try {
- return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, point: prepared.point || null, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };
+ return { ...metadata, ...await this.perform(tabId, action, {
+ ...message.params,
+ token,
+ url: prepared.url,
+ point: prepared.point || null,
+ checkable: prepared.checkable || null,
+ deadlineAt: message.actionDeadlineAt || 0,
+ }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };
} catch (error) {
// A transport failure can occur after trusted input was delivered.
return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, ...(error.dispatchState || {}), error: error.message };
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index ca34126ac..6708b81cc 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -2037,7 +2037,9 @@
// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)
if (el.tagName !== 'SELECT') {
if (actionDeadlineExpired()) return deadlineFailure();
- el.scrollIntoView({ behavior: 'smooth', block: 'center' });
+ // BiDi validates the target in the same turn. A smooth scroll leaves a
+ // transient offscreen geometry window where that validation must fail.
+ el.scrollIntoView({ behavior: params._bidiPrepare ? 'instant' : 'smooth', block: 'center' });
}
// Occlusion hit-test: for text/selector/index clicks, verify that the
@@ -5182,7 +5184,13 @@
const canonicalTargetName = _axCanonicalName(el);
const targetName = canonicalTargetName || _axAccessibleName(el);
if (!_isFullyVisibleForInteraction(el)) {
- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}
+ try {
+ el.scrollIntoView({
+ block: 'center',
+ inline: 'center',
+ ...(msg.params?._bidiPrepare ? { behavior: 'instant' } : {}),
+ });
+ } catch {}
}
try { el.focus({ preventScroll: true }); } catch {}
const rect = el.getBoundingClientRect();
@@ -5314,7 +5322,20 @@
{ deadlineExpired: true, retryable: true },
);
}
- if (msg.params?._bidiPrepare) return { ...prepareBidiTarget(el, msg.params._bidiPrepare), _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };
+ if (msg.params?._bidiPrepare) {
+ return {
+ ...prepareBidiTarget(el, msg.params._bidiPrepare),
+ ...(nativeCheckable ? {
+ checkable: {
+ inputType,
+ checkedBefore,
+ desiredChecked: inputType === 'radio' ? true : !checkedBefore,
+ checkboxIdentity: _axCheckboxIdentity(el, ref_id),
+ },
+ } : {}),
+ _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId,
+ };
+ }
dispatched = true;
const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());
if (filePickerGuard.blocked) {
diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjs
index 74bfd51db..9e2354311 100644
--- a/test/firefox-bidi-e2e.mjs
+++ b/test/firefox-bidi-e2e.mjs
@@ -72,6 +72,27 @@ try {
assert.deepEqual(pointPrepared.point,{x:22,y:210});
await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});
assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});
+ await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault();`);
+ const checked = await session.perform(runId, 'click', {
+ ...await mark('#checkable'),
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'checkable' },
+ });
+ assert.equal(checked.success, true);
+ assert.equal(checked.verified, true);
+ assert.equal(checked.checkedAfter, true);
+ const blockedCheck = await session.perform(runId, 'click', {
+ ...await mark('#blocked-checkable'),
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'blocked-checkable' },
+ });
+ assert.equal(blockedCheck.success, false);
+ assert.equal(blockedCheck.verified, false);
+ assert.equal(blockedCheck.noProgress, true);
+ await evaluate(`document.body.style.minHeight='3000px'; document.body.insertAdjacentHTML('beforeend', '<button id="below-fold" style="position:absolute;top:2600px">Below fold</button>'); document.querySelector('#below-fold').onclick=e=>window.belowFoldTrusted=e.isTrusted;`);
+ const belowFoldToken = crypto.randomUUID();
+ const belowFoldPrepared = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content',action:'click',params:{selector:'#below-fold',_bidiPrepare:${JSON.stringify(belowFoldToken)}}}).then(value=>JSON.stringify(value))`);
+ assert.equal(JSON.parse(belowFoldPrepared.value).bidiPrepared, true);
+ assert.equal((await session.perform(runId, 'click', { token: belowFoldToken, url })).success, true);
+ assert.equal((await evaluate('window.belowFoldTrusted')).value, true);
const obscured = await mark('#point-canvas');
await evaluate(`document.body.insertAdjacentHTML('beforeend','<div id="point-cover" style="position:fixed;left:20px;top:205px;width:10px;height:10px;background:red;z-index:99999"></div>')`);
await assert.rejects(session.perform(runId,'click',{...obscured,point:{x:22,y:210}}), error=>error.dispatchState.noDispatch===true && /covered/.test(error.message));
@@ -102,7 +123,7 @@ try {
// Invoke the real client in the packaged extension with a transport stub; no
// native manifest is installed in the user's normal Firefox configuration.
await session.perform(runId, 'key', { ...await mark('#text'), key: 'Tab', repeat: 2 });
- assert.equal((await evaluate('document.activeElement.id')).value, 'multiline');
+ assert.equal((await evaluate('document.activeElement.id')).value, 'checkable');
const deferred = await extensionEval(`(async () => {
const {FirefoxBidiClient} = await import(${JSON.stringify(extensionOrigin + '/src/bidi/client.js')});
const client = new FirefoxBidiClient(browser);
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 121c65042..67623d130 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -202,6 +202,37 @@ test('validation failures preserve no-dispatch status across the native client',
assert.equal(result.outcomeUnknown,false);
assert.equal(result.retryable,true);
});
+test('click_ax preparation carries checkable state into trusted input', async () => {
+ const client = new FirefoxBidiClient({
+ tabs: { sendMessage: async () => ({
+ bidiPrepared: true,
+ url: 'https://example.com/',
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'choice' },
+ }) },
+ });
+ client.runs.set(1, { runId: id(), bound: true });
+ let payload;
+ client.perform = async (_tabId, _action, received) => { payload = received; return { success: true, verified: true }; };
+ const result = await client.sendContent(1, { action: 'click_ax', params: { ref_id: 'ax-1' } });
+ assert.equal(result.success, true);
+ assert.deepEqual(payload.checkable, { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'choice' });
+});
+test('trusted checkable clicks report a prevented state transition', async () => {
+ const session = new BidiSession(); const runId = id(); const sent = [];
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });
+ session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type') ? false : true } });
+ session.send = async (method, params) => { sent.push({ method, params }); return {}; };
+ const result = await session.perform(runId, 'click', {
+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },
+ });
+ assert.equal(sent.some(item => item.method === 'input.performActions'), true);
+ assert.equal(result.success, false);
+ assert.equal(result.verified, false);
+ assert.equal(result.noProgress, true);
+ assert.equal(result.checkedAfter, false);
+ assert.equal(result.checkedChanged, false);
+});
test('post-dispatch failure and transport uncertainty are never safe retries', async () => {
const session = new BidiSession(); const runId = id();
session.runs.set(runId, {context:'tab'});