Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fence Firefox fallback uploads against stale page feedback
Barack Sokullu1db4cf59cfb7
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 4e255193a..78b700d84 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -1035,6 +1035,7 @@sampleFormControls(op?.target);const layoutChanged = coordinatePreparationShifted(op);if (!active || !op || layoutChanged || domTimer || unreported+ || (params.element && op.target !== params.element)|| lastUserAt > op.userAt|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {const error = new Error('Browser changed during action preparation. Re-observe before acting.');diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 034c81b36..d059cd110 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -1,4 +1,4 @@-import { installPageFeedback, beforePageAgentDispatch } from './page-feedback.js';+import { installPageFeedback, beforePageAgentDispatch, hasPageAgentDispatchOwner } from './page-feedback.js';import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js';import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';@@ -33482,7 +33482,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn { success: true, dispatched: false };})();`;- const buildInjectCode = actionDeadlineAt => `+ const buildInjectCode = (actionDeadlineAt, pageFeedbackGuard) => `(function() {const actionDeadlineAt = ${Number(actionDeadlineAt) || 0};const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;@@ -33522,6 +33522,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!(el instanceof HTMLInputElement) || el.type !== 'file') {return { success: false, dispatched: false, error: 'Selector does not match an <input type="file"> element: ' + selector };}+ const pageFeedbackGuard = ${JSON.stringify(pageFeedbackGuard || null)};+ const pageMonitor = window.__wbPageMonitor;+ if (pageFeedbackGuard?.operationId) {+ if (typeof pageMonitor?.activatePreparedDispatch !== 'function'+ || typeof pageMonitor?.withPreparedDispatch !== 'function') {+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,+ error: 'The page monitor is unavailable at the upload dispatch boundary. Re-observe the page before acting.' };+ }+ try {+ pageMonitor.activatePreparedDispatch({ operationId: pageFeedbackGuard.operationId, kind: 'input', element: el });+ } catch (e) {+ if (e?.code === 'page_feedback_pending') {+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,+ error: e.message || 'The browser changed during upload preparation. Re-observe the page before acting.' };+ }+ throw e;+ }+ }const compactIdentity = value => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, 160);let targetLabel = '';try {@@ -33554,10 +33572,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (deadlineExpired()) {return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };}- el.files = dt.files;- dispatched = true;- el.dispatchEvent(new Event('input', { bubbles: true }));- el.dispatchEvent(new Event('change', { bubbles: true }));+ const attach = () => {+ dispatched = true;+ el.files = dt.files;+ el.dispatchEvent(new Event('input', { bubbles: true }));+ el.dispatchEvent(new Event('change', { bubbles: true }));+ };+ if (pageFeedbackGuard?.operationId) pageMonitor.withPreparedDispatch(pageFeedbackGuard.operationId, attach);+ else attach();const attachmentState = el.files && el.files.length? 'input_attached': 'page_consumed';@@ -33616,10 +33638,23 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// change events, so a lost response must be treated as an unknown// upload outcome rather than retried blindly.this._throwIfAborted(abortSignal);+ const pageFeedbackGuard = await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {+ kind: 'input', selector: args.selector, fenceOnly: true,+ });+ this._throwIfAborted(abortSignal);+ if (hasPageAgentDispatchOwner(tabId) && !pageFeedbackGuard?.operationId) {+ return {+ success: false,+ dispatched: false,+ noDispatch: true,+ pageFeedbackPending: true,+ error: 'The page monitor could not validate the upload target at dispatch. Re-observe the page before acting.',+ };+ }markContentPipelineDispatched();let results;try {- results = await browser.tabs.executeScript(tabId, { code: buildInjectCode(actionDeadlineAt) });+ results = await browser.tabs.executeScript(tabId, { code: buildInjectCode(actionDeadlineAt, pageFeedbackGuard) });} catch (e) {this._throwIfAborted(abortSignal);return {@@ -33646,6 +33681,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout derror: res.error || 'Upload action deadline expired before dispatch',};}+ if (res?.pageFeedbackPending && res?.noDispatch === true) {+ contentPipelineDispatchState.started = false;+ return {+ ...res,+ success: false,+ dispatched: false,+ noDispatch: true,+ outcomeUnknown: false,+ retryable: true,+ };+ }this._throwIfAborted(abortSignal);if (!res) {return {diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 4e255193a..78b700d84 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -1035,6 +1035,7 @@sampleFormControls(op?.target);const layoutChanged = coordinatePreparationShifted(op);if (!active || !op || layoutChanged || domTimer || unreported+ || (params.element && op.target !== params.element)|| lastUserAt > op.userAt|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {const error = new Error('Browser changed during action preparation. Re-observe before acting.');diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex f71102148..e54444b4c 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -2693,6 +2693,52 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await session.close(); await browser.close(); }});+ test(`${build}: upload fences reject user changes and replacement inputs before file assignment`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const input = document.createElement('input'); input.id = 'guarded-upload'; input.type = 'file';+ document.body.append(input);+ });+ await page.waitForTimeout(200);+ const prepare = operationId => page.evaluate(id => {+ deliver('page_monitor_prepare', { operationId: id, tool: 'upload_file', selector: '#guarded-upload' });+ deliver('page_monitor_dispatch', { operationId: id, kind: 'input', selector: '#guarded-upload', fenceOnly: true });+ return lastMonitorResponse.guard;+ }, operationId);+ const assignIfCurrent = operationId => page.evaluate(id => {+ const input = document.getElementById('guarded-upload');+ const transfer = new DataTransfer(); transfer.items.add(new File(['x'], 'test.txt'));+ try {+ __wbPageMonitor.activatePreparedDispatch({ operationId: id, kind: 'input', element: input });+ } catch (error) {+ return { code: error.code, files: input.files.length };+ }+ __wbPageMonitor.withPreparedDispatch(id, () => { input.files = transfer.files; });+ return { code: null, files: input.files.length };+ }, operationId);++ const userGuard = await prepare('upload-after-user-change');+ assert.ok(userGuard?.operationId);+ await page.locator('#human').click();+ const afterUserChange = await assignIfCurrent('upload-after-user-change');+ assert.equal(afterUserChange.code, 'page_feedback_pending');+ assert.equal(afterUserChange.files, 0, 'user activity must block FileList assignment');++ const replacementGuard = await prepare('upload-after-target-replacement');+ assert.ok(replacementGuard?.operationId);+ await page.evaluate(() => {+ const oldInput = document.getElementById('guarded-upload');+ const replacement = document.createElement('input'); replacement.id = oldInput.id; replacement.type = 'file';+ oldInput.replaceWith(replacement);+ });+ await page.waitForTimeout(120);+ const afterReplacement = await assignIfCurrent('upload-after-target-replacement');+ assert.equal(afterReplacement.code, 'page_feedback_pending');+ assert.equal(afterReplacement.files, 0, 'a selector replacement must not receive a prepared upload');+ } finally { await browser.close(); }+ });+test(`${build}: navigation notes are plain text, deduplicated and survive history restoration`, async () => {const browser = await engine.launch({ headless: true });try {diff --git a/test/run.js b/test/run.jsindex 88925d913..0d6c703e1 100644--- a/test/run.js+++ b/test/run.js@@ -107389,6 +107389,113 @@ test('Firefox upload_file injects the exact user attachment bytes without re-fet}});+test('Firefox fallback upload aborts before FileList assignment when the prepared monitor target is stale', async () => {+ const originalBrowser = globalThis.browser;+ const messages = [];+ const scripts = [];+ class MockFileInput {+ constructor() { this.type = 'file'; this.id = 'upload'; this.labels = []; this.assignmentAttempts = 0; this._files = { length: 0 }; }+ get files() { return this._files; }+ set files(value) { this.assignmentAttempts++; this._files = value; }+ getAttribute() { return null; }+ dispatchEvent() { return true; }+ }+ const fileInput = new MockFileInput();+ class MockDataTransfer {+ constructor() {+ const files = [];+ files.item = index => files[index] || null;+ this.files = files;+ this.items = { add: file => { files.push(file); } };+ }+ }+ class MockFile {+ constructor(_bytes, name, options = {}) { this.name = name; this.type = options.type || ''; this.size = 1; }+ }+ const pageMonitor = {+ activatePreparedDispatch({ operationId, kind, element }) {+ assert.ok(operationId, 'the upload must carry its run-owned operation id');+ assert.equal(kind, 'input');+ assert.equal(element, fileInput, 'the final guard must bind to the probed file input');+ const error = new Error('Browser changed during upload preparation. Re-observe before acting.');+ error.code = 'page_feedback_pending';+ throw error;+ },+ withPreparedDispatch() { assert.fail('stale upload must not enter the assignment callback'); },+ };+ const browser = {+ tabs: {+ async get(tabId) { return { id: tabId, url: 'https://example.com/upload' }; },+ async sendMessage(_tabId, message) {+ messages.push(message);+ if (message.action === 'page_monitor_dispatch') {+ return { ready: true, guard: {+ operationId: message.params.operationId,+ runToken: message.params.runToken,+ documentToken: 'upload-document',+ revision: 12,+ } };+ }+ return { ready: true };+ },+ async executeScript(_tabId, details) {+ scripts.push(details.code);+ if (details.code.includes('WebBrain file attachment target probe')) {+ return [{ success: true, dispatched: false }];+ }+ if (details.code.includes('WebBrain file attachment settle probe')) {+ return [{ attachmentState: 'input_attached' }];+ }+ const document = {+ querySelectorAll(selector) {+ if (selector === 'input[type=file]') return [fileInput];+ return [];+ },+ };+ const result = vm.runInNewContext(details.code, {+ window: { __wbPageMonitor: pageMonitor },+ document,+ HTMLInputElement: MockFileInput,+ File: MockFile,+ DataTransfer: MockDataTransfer,+ Uint8Array,+ Event: class MockEvent {},+ });+ return [result];+ },+ },+ webNavigation: { async getAllFrames() { return [{ frameId: 0 }]; } },+ };+ let agent;+ try {+ globalThis.browser = browser;+ agent = new AgentFx({});+ await agent._beginPageFeedbackRun(42, 'chat');+ const [attachment] = agent._registerUserAttachments(42, [{+ kind: 'document', name: 'report.txt', mimeType: 'text/plain', dataUrl: 'data:text/plain;base64,cmVwb3J0',+ }]);+ const result = await agent.executeTool(42, 'upload_file', {+ selector: 'input[type=file]', attachmentId: attachment.attachmentId,+ }, null, { promptTier: 'mid' });++ assert.equal(result.success, false);+ assert.equal(result.pageFeedbackPending, true);+ assert.equal(result.noDispatch, true);+ assert.equal(result.dispatched, false);+ assert.equal(fileInput.assignmentAttempts, 0, 'stale feedback must prevent FileList assignment');+ assert.equal(fileInput.files.length, 0);+ assert.equal(scripts.length, 2, 'the stale injection must not run the settle probe');+ const boundary = messages.find(message => message.action === 'page_monitor_dispatch');+ assert.equal(boundary.params.kind, 'input');+ assert.equal(boundary.params.fenceOnly, true);+ assert.ok(scripts[1].includes('activatePreparedDispatch'), 'the page-side injection must revalidate the monitor guard');+ } finally {+ try { agent?._finishPageFeedbackRun(42); } catch {}+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});+test('Firefox upload_file deadline distinguishes target preparation from FileList dispatch', async () => {const originalBrowser = globalThis.browser;try {