Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: preserve Firefox BiDi typing and keyboard input

Emre Sokullu69835f471ae8
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index b79672ce1..c9d12972f 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -142,26 +142,28 @@ export class BidiSession {
if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;
el.removeAttribute('data-webbrain-bidi');
if (action === 'upload') return el.tagName === 'INPUT' && el.type === 'file';
- const r = el.getBoundingClientRect();
- const px = x === null ? r.x+r.width/2 : x;
- const py = y === null ? r.y+r.height/2 : y;
- if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;
- let hit = document.elementFromPoint(px, py);
- // elementFromPoint stops at each open shadow host. Descend through those
- // roots so trusted input accepts the same visible target as content.js.
- while (hit?.shadowRoot) {
- const inner = hit.shadowRoot.elementFromPoint(px, py);
- if (!inner || inner === hit) break;
- hit = inner;
- }
- const reaches = (node, target) => {
- while (node) {
- if (node === target) return true;
- node = node.parentNode || node.host;
+ if (action === 'click' || action === 'hover') {
+ const r = el.getBoundingClientRect();
+ const px = x === null ? r.x+r.width/2 : x;
+ const py = y === null ? r.y+r.height/2 : y;
+ if (px < 0 || py < 0 || px >= innerWidth || py >= innerHeight) return false;
+ let hit = document.elementFromPoint(px, py);
+ // elementFromPoint stops at each open shadow host. Descend through those
+ // roots so trusted input accepts the same visible target as content.js.
+ while (hit?.shadowRoot) {
+ const inner = hit.shadowRoot.elementFromPoint(px, py);
+ if (!inner || inner === hit) break;
+ hit = inner;
}
- return false;
- };
- if (!r.width || !r.height || !reaches(hit, el)) return false;
+ const reaches = (node, target) => {
+ while (node) {
+ if (node === target) return true;
+ node = node.parentNode || node.host;
+ }
+ return false;
+ };
+ if (!r.width || !r.height || !reaches(hit, el)) return false;
+ }
if (action === 'type' || action === 'field' || action === 'key') {
el.focus({preventScroll:true});
if (el.getRootNode().activeElement !== el) return false;
@@ -259,6 +261,19 @@ export class BidiSession {
const platform = await this.call(match, '() => navigator.platform');
await press('a', /Mac/.test(platform.result?.value || '') ? '\uE03D' : '\uE009');
await press('\uE003');
+ } else {
+ const positioned = await this.call(match, `(el) => {
+ if (!el.isConnected || el.getRootNode().activeElement !== el) return false;
+ if (el.isContentEditable) {
+ const selection = getSelection(); const range = document.createRange();
+ range.selectNodeContents(el); range.collapse(false);
+ selection.removeAllRanges(); selection.addRange(range);
+ } else if (typeof el.setSelectionRange === 'function') {
+ const end = el.value.length; el.setSelectionRange(end, end);
+ } else return false;
+ return true;
+ }`);
+ if (positioned.result?.value !== true) throw new Error('Could not position caret for append; no text sent');
}
// One character per dispatch bounds work after Stop and catches focus changes between keys.
for (const char of payload.text) {
diff --git a/test/firefox-bidi-e2e.mjs b/test/firefox-bidi-e2e.mjs
index 001ef4dd3..460a88a3e 100644
--- a/test/firefox-bidi-e2e.mjs
+++ b/test/firefox-bidi-e2e.mjs
@@ -51,6 +51,10 @@ try {
const typed = await session.perform(runId, 'field', { ...await mark('#text'), text: 'Hello Firefox', clear: true });
assert.equal(typed.verified, true);
assert.equal((await evaluate('document.querySelector("#text").value')).value, 'Hello Firefox');
+ await evaluate(`document.querySelector('#text').setSelectionRange(1, 1)`);
+ const appended = await session.perform(runId, 'type', { ...await mark('#text'), text: ' world' });
+ assert.equal(appended.verified, true);
+ assert.equal((await evaluate('document.querySelector("#text").value')).value, 'Hello Firefox world');
assert.equal((await evaluate('events.some(e => e[0] === "input" && e[1])')).value, true);
const uploaded = await session.perform(runId, 'upload', { ...await mark('#file'), filename: 'sample.txt', base64: Buffer.from('upload test').toString('base64') });
assert.equal(uploaded.success, true);
@@ -93,6 +97,8 @@ try {
assert.equal(JSON.parse(belowFoldPrepared.value).bidiPrepared, true);
assert.equal((await session.perform(runId, 'click', { token: belowFoldToken, url })).success, true);
assert.equal((await evaluate('window.belowFoldTrusted')).value, true);
+ await evaluate(`document.activeElement.blur()`);
+ assert.equal((await session.perform(runId, 'key', { ...await mark('body'), key: 'Escape' })).success, true);
const shadowToken = crypto.randomUUID();
await evaluate(`const host=document.createElement('div'); host.id='shadow-host'; host.style.cssText='position:fixed;left:20px;top:300px'; host.attachShadow({mode:'open'}).innerHTML='<button id="shadow-button">Shadow button</button>'; host.shadowRoot.querySelector('button').setAttribute('data-webbrain-bidi', ${JSON.stringify(shadowToken)}); host.shadowRoot.querySelector('button').onclick=e=>window.shadowTrusted=e.isTrusted; document.body.append(host);`);
const shadowNode = await session.send('script.evaluate', { target: { context }, expression: "document.querySelector('#shadow-host').shadowRoot.querySelector('#shadow-button')", awaitPromise: true });
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 7c8c7c4de..bad2c0312 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -243,6 +243,15 @@ test('trusted target validation descends through open shadow roots', async () =>
assert.match(validation, /shadowRoot\.elementFromPoint/);
assert.match(validation, /node\.parentNode \|\| node\.host/);
});
+test('keyboard validation does not require pointer geometry', async () => {
+ const session = new BidiSession(); const runId = id(); const calls = [];
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'body' } });
+ session.call = async (_match, fn) => { calls.push(fn); return { result: { value: true } }; };
+ session.send = async () => ({});
+ await session.perform(runId, 'key', { key: 'Escape' });
+ assert.match(calls[0], /action === 'click' \|\| action === 'hover'/);
+});
test('post-dispatch failure and transport uncertainty are never safe retries', async () => {
const session = new BidiSession(); const runId = id();
session.runs.set(runId, {context:'tab'});