Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: preserve Firefox BiDi typing and keyboard input
Emre Sokullu69835f471ae8
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex b79672ce1..c9d12972f 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -142,26 +142,28 @@ export class BidiSession {if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;el.removeAttribute('data-webbrain-bidi');if (action === 'upload') return el.tagName === 'INPUT' && el.type === 'file';- const r = el.getBoundingClientRect();- const px = x === null ? r.x+r.width/2 : x;- const py = y === null ? r.y+r.height/2 : y;- if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;- let hit = document.elementFromPoint(px, py);- // elementFromPoint stops at each open shadow host. Descend through those- // roots so trusted input accepts the same visible target as content.js.- while (hit?.shadowRoot) {- const inner = hit.shadowRoot.elementFromPoint(px, py);- if (!inner || inner === hit) break;- hit = inner;- }- const reaches = (node, target) => {- while (node) {- if (node === target) return true;- node = node.parentNode || node.host;+ if (action === 'click' || action === 'hover') {+ const r = el.getBoundingClientRect();+ const px = x === null ? r.x+r.width/2 : x;+ const py = y === null ? r.y+r.height/2 : y;+ if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;+ let hit = document.elementFromPoint(px, py);+ // elementFromPoint stops at each open shadow host. Descend through those+ // roots so trusted input accepts the same visible target as content.js.+ while (hit?.shadowRoot) {+ const inner = hit.shadowRoot.elementFromPoint(px, py);+ if (!inner || inner === hit) break;+ hit = inner;}- return false;- };- if (!r.width || !r.height || !reaches(hit, el)) return false;+ const reaches = (node, target) => {+ while (node) {+ if (node === target) return true;+ node = node.parentNode || node.host;+ }+ return false;+ };+ if (!r.width || !r.height || !reaches(hit, el)) return false;+ }if (action === 'type' || action === 'field' || action === 'key') {el.focus({preventScroll:true});if (el.getRootNode().activeElement !== el) return false;@@ -259,6 +261,19 @@ export class BidiSession {const platform = await this.call(match, '() => navigator.platform');await press('a', /Mac/.test(platform.result?.value || '') ? '\uE03D' : '\uE009');await press('\uE003');+ } else {+ const positioned = await this.call(match, `(el) => {+ if (!el.isConnected || el.getRootNode().activeElement !== el) return false;+ if (el.isContentEditable) {+ const selection = getSelection(); const range = document.createRange();+ range.selectNodeContents(el); range.collapse(false);+ selection.removeAllRanges(); selection.addRange(range);+ } else if (typeof el.setSelectionRange === 'function') {+ const end = el.value.length; el.setSelectionRange(end, end);+ } else return false;+ return true;+ }`);+ if (positioned.result?.value !== true) throw new Error('Could not position caret for append; no text sent');}// One character per dispatch bounds work after Stop and catches focus changes between keys.for (const char of payload.text) {diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjsindex 001ef4dd3..460a88a3e 100644--- a/test/firefox-bidi-e2e.mjs+++ b/test/firefox-bidi-e2e.mjs@@ -51,6 +51,10 @@ try {const typed = await session.perform(runId, 'field', { ...await mark('#text'), text: 'Hello Firefox', clear: true });assert.equal(typed.verified, true);assert.equal((await evaluate('document.querySelector("#text").value')).value, 'Hello Firefox');+ await evaluate(`document.querySelector('#text').setSelectionRange(1, 1)`);+ const appended = await session.perform(runId, 'type', { ...await mark('#text'), text: ' world' });+ assert.equal(appended.verified, true);+ assert.equal((await evaluate('document.querySelector("#text").value')).value, 'Hello Firefox world');assert.equal((await evaluate('events.some(e => e[0] === "input" && e[1])')).value, true);const uploaded = await session.perform(runId, 'upload', { ...await mark('#file'), filename: 'sample.txt', base64: Buffer.from('upload test').toString('base64') });assert.equal(uploaded.success, true);@@ -93,6 +97,8 @@ try {assert.equal(JSON.parse(belowFoldPrepared.value).bidiPrepared, true);assert.equal((await session.perform(runId, 'click', { token: belowFoldToken, url })).success, true);assert.equal((await evaluate('window.belowFoldTrusted')).value, true);+ await evaluate(`document.activeElement.blur()`);+ assert.equal((await session.perform(runId, 'key', { ...await mark('body'), key: 'Escape' })).success, true);const shadowToken = crypto.randomUUID();await evaluate(`const host=document.createElement('div'); host.id='shadow-host'; host.style.cssText='position:fixed;left:20px;top:300px'; host.attachShadow({mode:'open'}).innerHTML='<button id="shadow-button">Shadow button</button>'; host.shadowRoot.querySelector('button').setAttribute('data-webbrain-bidi', ${JSON.stringify(shadowToken)}); host.shadowRoot.querySelector('button').onclick=e=>window.shadowTrusted=e.isTrusted; document.body.append(host);`);const shadowNode = await session.send('script.evaluate', { target: { context }, expression: "document.querySelector('#shadow-host').shadowRoot.querySelector('#shadow-button')", awaitPromise: true });diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 7c8c7c4de..bad2c0312 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -243,6 +243,15 @@ test('trusted target validation descends through open shadow roots', async () =>assert.match(validation, /shadowRoot\.elementFromPoint/);assert.match(validation, /node\.parentNode \|\| node\.host/);});+test('keyboard validation does not require pointer geometry', async () => {+ const session = new BidiSession(); const runId = id(); const calls = [];+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'body' } });+ session.call = async (_match, fn) => { calls.push(fn); return { result: { value: true } }; };+ session.send = async () => ({});+ await session.perform(runId, 'key', { key: 'Escape' });+ assert.match(calls[0], /action === 'click' \|\| action === 'hover'/);+});test('post-dispatch failure and transport uncertainty are never safe retries', async () => {const session = new BidiSession(); const runId = id();session.runs.set(runId, {context:'tab'});