Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fence Firefox fallback uploads against stale page feedback

Barack Sokullu1db4cf59cfb7
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 4e255193a..78b700d84 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -1035,6 +1035,7 @@
sampleFormControls(op?.target);
const layoutChanged = coordinatePreparationShifted(op);
if (!active || !op || layoutChanged || domTimer || unreported
+ || (params.element && op.target !== params.element)
|| lastUserAt > op.userAt
|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 034c81b36..d059cd110 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -1,4 +1,4 @@
-import { installPageFeedback, beforePageAgentDispatch } from './page-feedback.js';
+import { installPageFeedback, beforePageAgentDispatch, hasPageAgentDispatchOwner } from './page-feedback.js';
import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
@@ -33482,7 +33482,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return { success: true, dispatched: false };
})();
`;
- const buildInjectCode = actionDeadlineAt => `
+ const buildInjectCode = (actionDeadlineAt, pageFeedbackGuard) => `
(function() {
const actionDeadlineAt = ${Number(actionDeadlineAt) || 0};
const deadlineExpired = () => actionDeadlineAt > 0 && Date.now() >= actionDeadlineAt;
@@ -33522,6 +33522,24 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!(el instanceof HTMLInputElement) || el.type !== 'file') {
return { success: false, dispatched: false, error: 'Selector does not match an <input type="file"> element: ' + selector };
}
+ const pageFeedbackGuard = ${JSON.stringify(pageFeedbackGuard || null)};
+ const pageMonitor = window.__wbPageMonitor;
+ if (pageFeedbackGuard?.operationId) {
+ if (typeof pageMonitor?.activatePreparedDispatch !== 'function'
+ || typeof pageMonitor?.withPreparedDispatch !== 'function') {
+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,
+ error: 'The page monitor is unavailable at the upload dispatch boundary. Re-observe the page before acting.' };
+ }
+ try {
+ pageMonitor.activatePreparedDispatch({ operationId: pageFeedbackGuard.operationId, kind: 'input', element: el });
+ } catch (e) {
+ if (e?.code === 'page_feedback_pending') {
+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,
+ error: e.message || 'The browser changed during upload preparation. Re-observe the page before acting.' };
+ }
+ throw e;
+ }
+ }
const compactIdentity = value => String(value || '').replace(/\\s+/g, ' ').trim().slice(0, 160);
let targetLabel = '';
try {
@@ -33554,10 +33572,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (deadlineExpired()) {
return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };
}
- el.files = dt.files;
- dispatched = true;
- el.dispatchEvent(new Event('input', { bubbles: true }));
- el.dispatchEvent(new Event('change', { bubbles: true }));
+ const attach = () => {
+ dispatched = true;
+ el.files = dt.files;
+ el.dispatchEvent(new Event('input', { bubbles: true }));
+ el.dispatchEvent(new Event('change', { bubbles: true }));
+ };
+ if (pageFeedbackGuard?.operationId) pageMonitor.withPreparedDispatch(pageFeedbackGuard.operationId, attach);
+ else attach();
const attachmentState = el.files && el.files.length
? 'input_attached'
: 'page_consumed';
@@ -33616,10 +33638,23 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// change events, so a lost response must be treated as an unknown
// upload outcome rather than retried blindly.
this._throwIfAborted(abortSignal);
+ const pageFeedbackGuard = await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {
+ kind: 'input', selector: args.selector, fenceOnly: true,
+ });
+ this._throwIfAborted(abortSignal);
+ if (hasPageAgentDispatchOwner(tabId) && !pageFeedbackGuard?.operationId) {
+ return {
+ success: false,
+ dispatched: false,
+ noDispatch: true,
+ pageFeedbackPending: true,
+ error: 'The page monitor could not validate the upload target at dispatch. Re-observe the page before acting.',
+ };
+ }
markContentPipelineDispatched();
let results;
try {
- results = await browser.tabs.executeScript(tabId, { code: buildInjectCode(actionDeadlineAt) });
+ results = await browser.tabs.executeScript(tabId, { code: buildInjectCode(actionDeadlineAt, pageFeedbackGuard) });
} catch (e) {
this._throwIfAborted(abortSignal);
return {
@@ -33646,6 +33681,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
error: res.error || 'Upload action deadline expired before dispatch',
};
}
+ if (res?.pageFeedbackPending && res?.noDispatch === true) {
+ contentPipelineDispatchState.started = false;
+ return {
+ ...res,
+ success: false,
+ dispatched: false,
+ noDispatch: true,
+ outcomeUnknown: false,
+ retryable: true,
+ };
+ }
this._throwIfAborted(abortSignal);
if (!res) {
return {
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 4e255193a..78b700d84 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -1035,6 +1035,7 @@
sampleFormControls(op?.target);
const layoutChanged = coordinatePreparationShifted(op);
if (!active || !op || layoutChanged || domTimer || unreported
+ || (params.element && op.target !== params.element)
|| lastUserAt > op.userAt
|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index f71102148..e54444b4c 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -2693,6 +2693,52 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await session.close(); await browser.close(); }
});
+ test(`${build}: upload fences reject user changes and replacement inputs before file assignment`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const input = document.createElement('input'); input.id = 'guarded-upload'; input.type = 'file';
+ document.body.append(input);
+ });
+ await page.waitForTimeout(200);
+ const prepare = operationId => page.evaluate(id => {
+ deliver('page_monitor_prepare', { operationId: id, tool: 'upload_file', selector: '#guarded-upload' });
+ deliver('page_monitor_dispatch', { operationId: id, kind: 'input', selector: '#guarded-upload', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ }, operationId);
+ const assignIfCurrent = operationId => page.evaluate(id => {
+ const input = document.getElementById('guarded-upload');
+ const transfer = new DataTransfer(); transfer.items.add(new File(['x'], 'test.txt'));
+ try {
+ __wbPageMonitor.activatePreparedDispatch({ operationId: id, kind: 'input', element: input });
+ } catch (error) {
+ return { code: error.code, files: input.files.length };
+ }
+ __wbPageMonitor.withPreparedDispatch(id, () => { input.files = transfer.files; });
+ return { code: null, files: input.files.length };
+ }, operationId);
+
+ const userGuard = await prepare('upload-after-user-change');
+ assert.ok(userGuard?.operationId);
+ await page.locator('#human').click();
+ const afterUserChange = await assignIfCurrent('upload-after-user-change');
+ assert.equal(afterUserChange.code, 'page_feedback_pending');
+ assert.equal(afterUserChange.files, 0, 'user activity must block FileList assignment');
+
+ const replacementGuard = await prepare('upload-after-target-replacement');
+ assert.ok(replacementGuard?.operationId);
+ await page.evaluate(() => {
+ const oldInput = document.getElementById('guarded-upload');
+ const replacement = document.createElement('input'); replacement.id = oldInput.id; replacement.type = 'file';
+ oldInput.replaceWith(replacement);
+ });
+ await page.waitForTimeout(120);
+ const afterReplacement = await assignIfCurrent('upload-after-target-replacement');
+ assert.equal(afterReplacement.code, 'page_feedback_pending');
+ assert.equal(afterReplacement.files, 0, 'a selector replacement must not receive a prepared upload');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: navigation notes are plain text, deduplicated and survive history restoration`, async () => {
const browser = await engine.launch({ headless: true });
try {
diff --git a/test/run.js b/test/run.js
index 88925d913..0d6c703e1 100644
--- a/test/run.js
+++ b/test/run.js
@@ -107389,6 +107389,113 @@ test('Firefox upload_file injects the exact user attachment bytes without re-fet
}
});
+test('Firefox fallback upload aborts before FileList assignment when the prepared monitor target is stale', async () => {
+ const originalBrowser = globalThis.browser;
+ const messages = [];
+ const scripts = [];
+ class MockFileInput {
+ constructor() { this.type = 'file'; this.id = 'upload'; this.labels = []; this.assignmentAttempts = 0; this._files = { length: 0 }; }
+ get files() { return this._files; }
+ set files(value) { this.assignmentAttempts++; this._files = value; }
+ getAttribute() { return null; }
+ dispatchEvent() { return true; }
+ }
+ const fileInput = new MockFileInput();
+ class MockDataTransfer {
+ constructor() {
+ const files = [];
+ files.item = index => files[index] || null;
+ this.files = files;
+ this.items = { add: file => { files.push(file); } };
+ }
+ }
+ class MockFile {
+ constructor(_bytes, name, options = {}) { this.name = name; this.type = options.type || ''; this.size = 1; }
+ }
+ const pageMonitor = {
+ activatePreparedDispatch({ operationId, kind, element }) {
+ assert.ok(operationId, 'the upload must carry its run-owned operation id');
+ assert.equal(kind, 'input');
+ assert.equal(element, fileInput, 'the final guard must bind to the probed file input');
+ const error = new Error('Browser changed during upload preparation. Re-observe before acting.');
+ error.code = 'page_feedback_pending';
+ throw error;
+ },
+ withPreparedDispatch() { assert.fail('stale upload must not enter the assignment callback'); },
+ };
+ const browser = {
+ tabs: {
+ async get(tabId) { return { id: tabId, url: 'https://example.com/upload' }; },
+ async sendMessage(_tabId, message) {
+ messages.push(message);
+ if (message.action === 'page_monitor_dispatch') {
+ return { ready: true, guard: {
+ operationId: message.params.operationId,
+ runToken: message.params.runToken,
+ documentToken: 'upload-document',
+ revision: 12,
+ } };
+ }
+ return { ready: true };
+ },
+ async executeScript(_tabId, details) {
+ scripts.push(details.code);
+ if (details.code.includes('WebBrain file attachment target probe')) {
+ return [{ success: true, dispatched: false }];
+ }
+ if (details.code.includes('WebBrain file attachment settle probe')) {
+ return [{ attachmentState: 'input_attached' }];
+ }
+ const document = {
+ querySelectorAll(selector) {
+ if (selector === 'input[type=file]') return [fileInput];
+ return [];
+ },
+ };
+ const result = vm.runInNewContext(details.code, {
+ window: { __wbPageMonitor: pageMonitor },
+ document,
+ HTMLInputElement: MockFileInput,
+ File: MockFile,
+ DataTransfer: MockDataTransfer,
+ Uint8Array,
+ Event: class MockEvent {},
+ });
+ return [result];
+ },
+ },
+ webNavigation: { async getAllFrames() { return [{ frameId: 0 }]; } },
+ };
+ let agent;
+ try {
+ globalThis.browser = browser;
+ agent = new AgentFx({});
+ await agent._beginPageFeedbackRun(42, 'chat');
+ const [attachment] = agent._registerUserAttachments(42, [{
+ kind: 'document', name: 'report.txt', mimeType: 'text/plain', dataUrl: 'data:text/plain;base64,cmVwb3J0',
+ }]);
+ const result = await agent.executeTool(42, 'upload_file', {
+ selector: 'input[type=file]', attachmentId: attachment.attachmentId,
+ }, null, { promptTier: 'mid' });
+
+ assert.equal(result.success, false);
+ assert.equal(result.pageFeedbackPending, true);
+ assert.equal(result.noDispatch, true);
+ assert.equal(result.dispatched, false);
+ assert.equal(fileInput.assignmentAttempts, 0, 'stale feedback must prevent FileList assignment');
+ assert.equal(fileInput.files.length, 0);
+ assert.equal(scripts.length, 2, 'the stale injection must not run the settle probe');
+ const boundary = messages.find(message => message.action === 'page_monitor_dispatch');
+ assert.equal(boundary.params.kind, 'input');
+ assert.equal(boundary.params.fenceOnly, true);
+ assert.ok(scripts[1].includes('activatePreparedDispatch'), 'the page-side injection must revalidate the monitor guard');
+ } finally {
+ try { agent?._finishPageFeedbackRun(42); } catch {}
+ if (originalBrowser === undefined) delete globalThis.browser;
+ else globalThis.browser = originalBrowser;
+ }
+});
+
test('Firefox upload_file deadline distinguishes target preparation from FileList dispatch', async () => {
const originalBrowser = globalThis.browser;
try {