Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: verify Firefox BiDi trusted checkable clicks
Emre Sokullua4881b12f658
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 364a93f3c..a1ed9b363 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -173,6 +173,10 @@ export class BidiSession {}const typing = action === 'type' || action === 'field';const clear = action === 'field' ? payload.clear !== false : payload.clear === true;+ const checkable = action === 'click' && payload.checkable;+ if (checkable && (!['checkbox', 'radio'].includes(checkable.inputType) || typeof checkable.desiredChecked !== 'boolean')) {+ throw new Error('Invalid checkable target metadata');+ }const before = typing ? await this.call(match, '(el) => el.isContentEditable ? el.innerText : el.value') : null;const assertFocus = async () => {assertLive();@@ -202,6 +206,35 @@ export class BidiSession {try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }assertLive();+ if (checkable) {+ await new Promise(resolve => setTimeout(resolve, 80));+ assertLive();+ const checked = await this.call(match, '(el) => el.isConnected && (el.type === "checkbox" || el.type === "radio") ? !!el.checked : null');+ const checkedAfter = checked.result?.value;+ if (typeof checkedAfter !== 'boolean') {+ return { success: false, dispatched: true, outcomeUnknown: true, retryable: false,+ error: 'Checkable target changed after trusted click; inspect the page before retrying.' };+ }+ const stateMatchesDesired = checkedAfter === checkable.desiredChecked;+ return {+ success: stateMatchesDesired,+ dispatched: true,+ verified: stateMatchesDesired,+ checkedBefore: !!checkable.checkedBefore,+ checkedAfter,+ checkedChanged: !!checkable.checkedBefore !== checkedAfter,+ desiredChecked: checkable.desiredChecked,+ checkboxIdentity: checkable.checkboxIdentity,+ checkboxState: { identity: checkable.checkboxIdentity, desiredChecked: checkable.desiredChecked, actualChecked: checkedAfter },+ ...(stateMatchesDesired ? { observedEffects: ['checked_state'] } : {+ noProgress: true,+ error: checkable.inputType === 'checkbox'+ ? `Checkbox remained ${checkedAfter ? 'checked' : 'unchecked'} after trusted click.`+ : 'Radio remained unselected after trusted click.',+ }),+ method: 'firefox-bidi',+ };+ }} else if (typing) {if (typeof payload.text !== 'string' || payload.text.length > 10000) throw new Error('Trusted text input is limited to 10000 characters per action');if (/[\r\n]/.test(payload.text)) {diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex 516c8d984..02f077ddf 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -109,7 +109,14 @@ export class FirefoxBidiClient {if (message.actionDeadlineAt && Date.now() >= message.actionDeadlineAt) return { success: false, dispatched: false, noDispatch: true, deadlineExpired: true };const metadata = { fieldMeta: prepared.fieldMeta, ...(message.params?.ref_id ? { ref_id: message.params.ref_id } : {}) };try {- return { ...metadata, ...await this.perform(tabId, action, { ...message.params, token, url: prepared.url, point: prepared.point || null, deadlineAt: message.actionDeadlineAt || 0 }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };+ return { ...metadata, ...await this.perform(tabId, action, {+ ...message.params,+ token,+ url: prepared.url,+ point: prepared.point || null,+ checkable: prepared.checkable || null,+ deadlineAt: message.actionDeadlineAt || 0,+ }), ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };} catch (error) {// A transport failure can occur after trusted input was delivered.return { ...metadata, success: false, dispatched: true, outcomeUnknown: true, retryable: false, ...(error.dispatchState || {}), error: error.message };diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex ca34126ac..6708b81cc 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2037,7 +2037,9 @@// Do NOT scrollIntoView on SELECT elements (hidden selects in modals cause scroll jumps)if (el.tagName !== 'SELECT') {if (actionDeadlineExpired()) return deadlineFailure();- el.scrollIntoView({ behavior: 'smooth', block: 'center' });+ // BiDi validates the target in the same turn. A smooth scroll leaves a+ // transient offscreen geometry window where that validation must fail.+ el.scrollIntoView({ behavior: params._bidiPrepare ? 'instant' : 'smooth', block: 'center' });}// Occlusion hit-test: for text/selector/index clicks, verify that the@@ -5182,7 +5184,13 @@const canonicalTargetName = _axCanonicalName(el);const targetName = canonicalTargetName || _axAccessibleName(el);if (!_isFullyVisibleForInteraction(el)) {- try { el.scrollIntoView({ block: 'center', inline: 'center' }); } catch {}+ try {+ el.scrollIntoView({+ block: 'center',+ inline: 'center',+ ...(msg.params?._bidiPrepare ? { behavior: 'instant' } : {}),+ });+ } catch {}}try { el.focus({ preventScroll: true }); } catch {}const rect = el.getBoundingClientRect();@@ -5314,7 +5322,20 @@{ deadlineExpired: true, retryable: true },);}- if (msg.params?._bidiPrepare) return { ...prepareBidiTarget(el, msg.params._bidiPrepare), _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId };+ if (msg.params?._bidiPrepare) {+ return {+ ...prepareBidiTarget(el, msg.params._bidiPrepare),+ ...(nativeCheckable ? {+ checkable: {+ inputType,+ checkedBefore,+ desiredChecked: inputType === 'radio' ? true : !checkedBefore,+ checkboxIdentity: _axCheckboxIdentity(el, ref_id),+ },+ } : {}),+ _filePickerGuardId: clickWithoutNativeFilePicker(() => {}).guardId,+ };+ }dispatched = true;const filePickerGuard = clickWithoutNativeFilePicker(() => el.click());if (filePickerGuard.blocked) {diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjsindex 74bfd51db..9e2354311 100644--- a/test/firefox-bidi-e2e.mjs+++ b/test/firefox-bidi-e2e.mjs@@ -72,6 +72,27 @@ try {assert.deepEqual(pointPrepared.point,{x:22,y:210});await session.perform(runId,'click',{token:pointToken,url,point:pointPrepared.point});assert.deepEqual(JSON.parse((await evaluate('JSON.stringify(window.clickedPoint)')).value),{x:22,y:210,trusted:true});+ await evaluate(`document.body.insertAdjacentHTML('beforeend', '<input id="checkable" type="checkbox"><input id="blocked-checkable" type="checkbox">'); document.querySelector('#blocked-checkable').onclick=event=>event.preventDefault();`);+ const checked = await session.perform(runId, 'click', {+ ...await mark('#checkable'),+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'checkable' },+ });+ assert.equal(checked.success, true);+ assert.equal(checked.verified, true);+ assert.equal(checked.checkedAfter, true);+ const blockedCheck = await session.perform(runId, 'click', {+ ...await mark('#blocked-checkable'),+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'blocked-checkable' },+ });+ assert.equal(blockedCheck.success, false);+ assert.equal(blockedCheck.verified, false);+ assert.equal(blockedCheck.noProgress, true);+ await evaluate(`document.body.style.minHeight='3000px'; document.body.insertAdjacentHTML('beforeend', '<button id="below-fold" style="position:absolute;top:2600px">Below fold</button>'); document.querySelector('#below-fold').onclick=e=>window.belowFoldTrusted=e.isTrusted;`);+ const belowFoldToken = crypto.randomUUID();+ const belowFoldPrepared = await extensionEval(`browser.tabs.sendMessage(${tabId}, {target:'content',action:'click',params:{selector:'#below-fold',_bidiPrepare:${JSON.stringify(belowFoldToken)}}}).then(value=>JSON.stringify(value))`);+ assert.equal(JSON.parse(belowFoldPrepared.value).bidiPrepared, true);+ assert.equal((await session.perform(runId, 'click', { token: belowFoldToken, url })).success, true);+ assert.equal((await evaluate('window.belowFoldTrusted')).value, true);const obscured = await mark('#point-canvas');await evaluate(`document.body.insertAdjacentHTML('beforeend','<div id="point-cover" style="position:fixed;left:20px;top:205px;width:10px;height:10px;background:red;z-index:99999"></div>')`);await assert.rejects(session.perform(runId,'click',{...obscured,point:{x:22,y:210}}), error=>error.dispatchState.noDispatch===true && /covered/.test(error.message));@@ -102,7 +123,7 @@ try {// Invoke the real client in the packaged extension with a transport stub; no// native manifest is installed in the user's normal Firefox configuration.await session.perform(runId, 'key', { ...await mark('#text'), key: 'Tab', repeat: 2 });- assert.equal((await evaluate('document.activeElement.id')).value, 'multiline');+ assert.equal((await evaluate('document.activeElement.id')).value, 'checkable');const deferred = await extensionEval(`(async () => {const {FirefoxBidiClient} = await import(${JSON.stringify(extensionOrigin + '/src/bidi/client.js')});const client = new FirefoxBidiClient(browser);diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 121c65042..67623d130 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -202,6 +202,37 @@ test('validation failures preserve no-dispatch status across the native client',assert.equal(result.outcomeUnknown,false);assert.equal(result.retryable,true);});+test('click_ax preparation carries checkable state into trusted input', async () => {+ const client = new FirefoxBidiClient({+ tabs: { sendMessage: async () => ({+ bidiPrepared: true,+ url: 'https://example.com/',+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'choice' },+ }) },+ });+ client.runs.set(1, { runId: id(), bound: true });+ let payload;+ client.perform = async (_tabId, _action, received) => { payload = received; return { success: true, verified: true }; };+ const result = await client.sendContent(1, { action: 'click_ax', params: { ref_id: 'ax-1' } });+ assert.equal(result.success, true);+ assert.deepEqual(payload.checkable, { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'choice' });+});+test('trusted checkable clicks report a prevented state transition', async () => {+ const session = new BidiSession(); const runId = id(); const sent = [];+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'check' } });+ session.call = async (_match, fn) => ({ result: { value: fn.includes('checkbox" || el.type') ? false : true } });+ session.send = async (method, params) => { sent.push({ method, params }); return {}; };+ const result = await session.perform(runId, 'click', {+ checkable: { inputType: 'checkbox', checkedBefore: false, desiredChecked: true, checkboxIdentity: 'check' },+ });+ assert.equal(sent.some(item => item.method === 'input.performActions'), true);+ assert.equal(result.success, false);+ assert.equal(result.verified, false);+ assert.equal(result.noProgress, true);+ assert.equal(result.checkedAfter, false);+ assert.equal(result.checkedChanged, false);+});test('post-dispatch failure and transport uncertainty are never safe retries', async () => {const session = new BidiSession(); const runId = id();session.runs.set(runId, {context:'tab'});