Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: attribute native uploads to the Firefox file input
Barack Sokullud94de37ac84e
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 96f56e8ff..8d37dd9ae 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -47,6 +47,8 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behaviorcover interventions during asynchronous preparation.- Both Chrome upload paths check feedback after preparation, immediately beforeattaching files. A skipped attachment retains its undispatched outcome.+- Firefox companion uploads register as input and place their native marker on+ the file input, preserving attribution when a different control holds focus.- Runtime observations use nonce-delimited `page_feedback` untrusted data. Theydo not enter the trusted text-steering authorization path. Screenshots followthe current capture, vision-routing and budget policies.diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 0c5076aed..3fcac5078 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -238,13 +238,13 @@ export class BidiSession {const guard = payload.pageFeedbackGuard;if (!guard) return; // Compatibility with clients without page monitoring.const marker = JSON.stringify({ ...guard, kind, sequence: ++nativeSequence });- const checked = await this.call(match, `(el, fence, marker, kind) => {+ const checked = await this.call(match, `(el, fence, marker, kind, action) => {if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;- const target = kind === 'input' ? el.getRootNode().activeElement || el : el;+ const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;target.setAttribute('data-webbrain-native-action', marker);return true;}`, [{ type: 'string', value: `${guard.documentToken}:${guard.revision}` },- { type: 'string', value: marker }, { type: 'string', value: kind }]);+ { type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);if (checked.result?.value !== true) {const error = new Error('Page changed during native preparation; no further input sent');error.code = 'page_feedback_pending'; throw error;diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex dfea9162f..318afaaad 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -143,7 +143,7 @@ export class FirefoxBidiClient {if (this.runs.get(tabId) !== owner || owner.disconnected || owner.signal?.aborted) throw new Error('Run stopped');const pageFeedbackGuard = await beforePageAgentDispatch(this.api, tabId, {kind: action === 'navigate' ? 'navigate' : action === 'scroll' ? 'scroll'- : ['type', 'field', 'key'].includes(action) ? 'input' : 'click',+ : ['type', 'field', 'key', 'upload'].includes(action) ? 'input' : 'click',...(payload.point ? { x: payload.point.x, y: payload.point.y } : {}),selector: payload.selector, ref_id: payload.ref_id, frameId: payload.frameId || 0,navigationCandidate: ['click', 'checked'].includes(action),diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 213becfbe..5dc2eaac8 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -1,9 +1,55 @@import { test } from 'node:test';import assert from 'node:assert/strict';+import vm from 'node:vm';import { FirefoxBidiClient } from '../src/firefox/src/bidi/client.js';import { BidiSession } from '../firefox-companion/session.mjs';+import { pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';const id = () => crypto.randomUUID();++test('uploads register input dispatches with the extension monitor', async () => {+ const tab = 955, owner = { ...pageFeedbackMethods, isRunning: () => true, _checkAbort: () => false };+ const guard = { documentToken: 'file-document', revision: 0, operationId: 'upload' };+ const registrations = [];+ const client = new FirefoxBidiClient({ tabs: { sendMessage: async (_tab, message) => {+ registrations.push(message.params); return { guard };+ } } });+ client.runs.set(tab, { runId: id(), bound: true });+ client.request = async (_command, args) => { assert.deepEqual(args.payload.pageFeedbackGuard, guard); return { success: true }; };+ await owner._beginPageFeedbackRun(tab, 'interactive');+ try {+ await client.perform(tab, 'upload', { selector: '#file' });+ assert.equal(registrations.length, 1);+ assert.equal(registrations[0].kind, 'input');+ assert.equal(registrations[0].navigationCandidate, false);+ } finally { owner._finishPageFeedbackRun(tab); }+});++test('an unfocused upload marks the file input instead of the active control', async () => {+ const session = new BidiSession(), runId = id(), attributes = new Map(), focusedAttributes = new Map();+ const guard = { documentToken: 'file-document', revision: 0, operationId: 'upload' };+ const token = id(), focused = { setAttribute: (name, value) => focusedAttributes.set(name, value) };+ const el = { isConnected: true, tagName: 'INPUT', type: 'file', files: [],+ getRootNode: () => ({ activeElement: focused }),+ getAttribute: name => name === 'data-webbrain-bidi' ? token : attributes.get(name),+ removeAttribute: name => attributes.delete(name), setAttribute: (name, value) => attributes.set(name, value) };+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'file' } });+ session.call = async (_match, declaration, args = []) => ({ result: { value: vm.runInNewContext(`(${declaration})`, {+ document: { documentElement: { getAttribute: () => 'file-document:0' } },+ })(el, ...args.map(arg => arg.value)) } });+ session.send = async method => {+ if (method === 'input.setFiles') {+ assert.equal(JSON.parse(attributes.get('data-webbrain-native-action')).kind, 'input');+ assert.equal(focusedAttributes.has('data-webbrain-native-action'), false);+ el.files = [{ name: 'test.txt', size: 1 }];+ }+ return {};+ };+ try {+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);+ } finally { await session.close(); }+});test('connection loss cannot fall through to synthetic input', async () => {let messages = 0;const client = new FirefoxBidiClient({ tabs: { sendMessage() { messages++; } } });diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex e3f15072f..99c1ebb66 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -4,6 +4,7 @@ import { test } from 'node:test';import { chromium, firefox } from 'playwright';import { CDPClient } from '../src/chrome/src/cdp/cdp-client.js';import { pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';+import { BidiSession } from '../firefox-companion/session.mjs';const read = (build, file) => fs.readFileSync(new URL(`../src/${build}/src/${file}`, import.meta.url), 'utf8');const html = `<!doctype html><style>@@ -445,6 +446,38 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: native uploads target an unfocused file input without hiding user input`, async () => {+ const { browser, page } = await fixture(engine, build);+ const session = new BidiSession(), runId = crypto.randomUUID(), token = crypto.randomUUID();+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => ({ context: 'tab', node: { sharedId: 'upload' } });+ session.call = async (_match, declaration, args = []) => ({ result: { value: await page.evaluate(({ declaration, args }) =>+ (0, eval)(`(${declaration})`)(document.getElementById('upload'), ...args.map(arg => arg.value)), { declaration, args }) } });+ session.send = async (method, params) => {+ if (method === 'input.setFiles') await page.locator('#upload').setInputFiles(params.files);+ return {};+ };+ try {+ await page.evaluate(token => {+ const input = document.createElement('input'); input.id = 'upload'; input.type = 'file';+ input.setAttribute('data-webbrain-bidi', token); document.body.append(input);+ }, token);+ await page.waitForTimeout(200);+ await page.locator('#field').focus();+ const guard = await page.evaluate(() => {+ deliver('page_monitor_prepare', { operationId: 'upload', tool: 'upload_file', selector: '#upload' });+ deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false });+ feedback = []; return lastMonitorResponse.guard;+ });+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);+ await page.waitForTimeout(200);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);+ assert.equal(await page.evaluate(() => document.activeElement.id), 'field');+ await page.keyboard.insertText('human');+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'input' && event.source === 'user'));+ } finally { await session.close(); await browser.close(); }+ });+test(`${build}: navigation notes are plain text, deduplicated and survive history restoration`, async () => {const browser = await engine.launch({ headless: true });try {