Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: attribute native uploads to the Firefox file input

Barack Sokullud94de37ac84e
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.md
index 96f56e8ff..8d37dd9ae 100644
--- a/docs/page-feedback-monitor.md
+++ b/docs/page-feedback-monitor.md
@@ -47,6 +47,8 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
cover interventions during asynchronous preparation.
- Both Chrome upload paths check feedback after preparation, immediately before
attaching files. A skipped attachment retains its undispatched outcome.
+- Firefox companion uploads register as input and place their native marker on
+ the file input, preserving attribution when a different control holds focus.
- Runtime observations use nonce-delimited `page_feedback` untrusted data. They
do not enter the trusted text-steering authorization path. Screenshots follow
the current capture, vision-routing and budget policies.
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 0c5076aed..3fcac5078 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -238,13 +238,13 @@ export class BidiSession {
const guard = payload.pageFeedbackGuard;
if (!guard) return; // Compatibility with clients without page monitoring.
const marker = JSON.stringify({ ...guard, kind, sequence: ++nativeSequence });
- const checked = await this.call(match, `(el, fence, marker, kind) => {
+ const checked = await this.call(match, `(el, fence, marker, kind, action) => {
if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;
- const target = kind === 'input' ? el.getRootNode().activeElement || el : el;
+ const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;
target.setAttribute('data-webbrain-native-action', marker);
return true;
}`, [{ type: 'string', value: `${guard.documentToken}:${guard.revision}` },
- { type: 'string', value: marker }, { type: 'string', value: kind }]);
+ { type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);
if (checked.result?.value !== true) {
const error = new Error('Page changed during native preparation; no further input sent');
error.code = 'page_feedback_pending'; throw error;
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index dfea9162f..318afaaad 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -143,7 +143,7 @@ export class FirefoxBidiClient {
if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) throw new Error('Run stopped');
const pageFeedbackGuard = await beforePageAgentDispatch(this.api, tabId, {
kind: action === 'navigate' ? 'navigate' : action === 'scroll' ? 'scroll'
- : ['type', 'field', 'key'].includes(action) ? 'input' : 'click',
+ : ['type', 'field', 'key', 'upload'].includes(action) ? 'input' : 'click',
...(payload.point ? { x: payload.point.x, y: payload.point.y } : {}),
selector: payload.selector, ref_id: payload.ref_id, frameId: payload.frameId || 0,
navigationCandidate: ['click', 'checked'].includes(action),
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 213becfbe..5dc2eaac8 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -1,9 +1,55 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
+import vm from 'node:vm';
import { FirefoxBidiClient } from '../src/firefox/src/bidi/client.js';
import { BidiSession } from '../firefox-companion/session.mjs';
+import { pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';
const id = () => crypto.randomUUID();
+
+test('uploads register input dispatches with the extension monitor', async () => {
+ const tab = 955, owner = { ...pageFeedbackMethods, isRunning: () => true, _checkAbort: () => false };
+ const guard = { documentToken: 'file-document', revision: 0, operationId: 'upload' };
+ const registrations = [];
+ const client = new FirefoxBidiClient({ tabs: { sendMessage: async (_tab, message) => {
+ registrations.push(message.params); return { guard };
+ } } });
+ client.runs.set(tab, { runId: id(), bound: true });
+ client.request = async (_command, args) => { assert.deepEqual(args.payload.pageFeedbackGuard, guard); return { success: true }; };
+ await owner._beginPageFeedbackRun(tab, 'interactive');
+ try {
+ await client.perform(tab, 'upload', { selector: '#file' });
+ assert.equal(registrations.length, 1);
+ assert.equal(registrations[0].kind, 'input');
+ assert.equal(registrations[0].navigationCandidate, false);
+ } finally { owner._finishPageFeedbackRun(tab); }
+});
+
+test('an unfocused upload marks the file input instead of the active control', async () => {
+ const session = new BidiSession(), runId = id(), attributes = new Map(), focusedAttributes = new Map();
+ const guard = { documentToken: 'file-document', revision: 0, operationId: 'upload' };
+ const token = id(), focused = { setAttribute: (name, value) => focusedAttributes.set(name, value) };
+ const el = { isConnected: true, tagName: 'INPUT', type: 'file', files: [],
+ getRootNode: () => ({ activeElement: focused }),
+ getAttribute: name => name === 'data-webbrain-bidi' ? token : attributes.get(name),
+ removeAttribute: name => attributes.delete(name), setAttribute: (name, value) => attributes.set(name, value) };
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'file' } });
+ session.call = async (_match, declaration, args = []) => ({ result: { value: vm.runInNewContext(`(${declaration})`, {
+ document: { documentElement: { getAttribute: () => 'file-document:0' } },
+ })(el, ...args.map(arg => arg.value)) } });
+ session.send = async method => {
+ if (method === 'input.setFiles') {
+ assert.equal(JSON.parse(attributes.get('data-webbrain-native-action')).kind, 'input');
+ assert.equal(focusedAttributes.has('data-webbrain-native-action'), false);
+ el.files = [{ name: 'test.txt', size: 1 }];
+ }
+ return {};
+ };
+ try {
+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);
+ } finally { await session.close(); }
+});
test('connection loss cannot fall through to synthetic input', async () => {
let messages = 0;
const client = new FirefoxBidiClient({ tabs: { sendMessage() { messages++; } } });
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index e3f15072f..99c1ebb66 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -4,6 +4,7 @@ import { test } from 'node:test';
import { chromium, firefox } from 'playwright';
import { CDPClient } from '../src/chrome/src/cdp/cdp-client.js';
import { pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';
+import { BidiSession } from '../firefox-companion/session.mjs';
const read = (build, file) => fs.readFileSync(new URL(`../src/${build}/src/${file}`, import.meta.url), 'utf8');
const html = `<!doctype html><style>
@@ -445,6 +446,38 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: native uploads target an unfocused file input without hiding user input`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ const session = new BidiSession(), runId = crypto.randomUUID(), token = crypto.randomUUID();
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'upload' } });
+ session.call = async (_match, declaration, args = []) => ({ result: { value: await page.evaluate(({ declaration, args }) =>
+ (0, eval)(`(${declaration})`)(document.getElementById('upload'), ...args.map(arg => arg.value)), { declaration, args }) } });
+ session.send = async (method, params) => {
+ if (method === 'input.setFiles') await page.locator('#upload').setInputFiles(params.files);
+ return {};
+ };
+ try {
+ await page.evaluate(token => {
+ const input = document.createElement('input'); input.id = 'upload'; input.type = 'file';
+ input.setAttribute('data-webbrain-bidi', token); document.body.append(input);
+ }, token);
+ await page.waitForTimeout(200);
+ await page.locator('#field').focus();
+ const guard = await page.evaluate(() => {
+ deliver('page_monitor_prepare', { operationId: 'upload', tool: 'upload_file', selector: '#upload' });
+ deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false });
+ feedback = []; return lastMonitorResponse.guard;
+ });
+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);
+ await page.waitForTimeout(200);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);
+ assert.equal(await page.evaluate(() => document.activeElement.id), 'field');
+ await page.keyboard.insertText('human');
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'input' && event.source === 'user'));
+ } finally { await session.close(); await browser.close(); }
+ });
+
test(`${build}: navigation notes are plain text, deduplicated and survive history restoration`, async () => {
const browser = await engine.launch({ headless: true });
try {