Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix: defer input attribution until actual dispatch

Barack Sokullu9615a2a2fed5
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.md
index 31692cb59..a3cea5d26 100644
--- a/docs/page-feedback-monitor.md
+++ b/docs/page-feedback-monitor.md
@@ -45,6 +45,9 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behavior
results have matching messages. In-flight results remain; undispatched sibling
calls receive explicit skipped results. Transport and local dispatch gates
cover interventions during asynchronous preparation.
+- Content and Firefox native preparation handshakes check the revision without
+ claiming input events. Local dispatch and native markers activate expectations
+ at the mutation boundary, preserving same-target intervention during preparation.
- Both Chrome upload paths check feedback after preparation, immediately before
attaching files. A skipped attachment retains its undispatched outcome.
- Firefox companion uploads register as input and place their native marker on
@@ -86,4 +89,9 @@ container scrolling, animation noise, cleanup/restart and persisted navigation
notes. Browser extension APIs are mocked in these fixtures; they are not a live
companion-session integration test.
+Visibility regressions include fixed-size ancestors changing class or CSS custom
+properties to reveal or hide non-interactive content. Native preparation fixtures
+also exercise the real companion code across an asynchronous target lookup while
+the user edits the same field.
+
The monitor regressions are also included in `npm run test:runtime-lifecycle`.
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 713a2ef16..9504debb1 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -40698,6 +40698,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
kind: /type|field|key/.test(name) ? 'input' : name === 'scroll' ? 'scroll' : /click|checked|hover|drag/.test(name) ? 'click' : 'dom',
frameId: messageOptions?.frameId || 0, selector: contentArgs?.selector, ref_id: contentArgs?.ref_id,
navigationCandidate: ['click', 'click_ax', 'set_checked'].includes(name),
+ fenceOnly: true,
});
}
if (earlyCdpAbortSignal) {
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index c5fc8beed..80442de92 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -188,7 +188,11 @@
for (const record of records) {
if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
- for (const el of [...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100))
+ const descendants = new Set([
+ ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),
+ ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),
+ ]);
+ for (const el of descendants)
changes.push({ type: 'layout', target: el });
}
let measured = 0;
@@ -391,7 +395,7 @@
}
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
- op.until = Date.now() + 1500; op.dispatched = true;
+ op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;
operations.set(op.operationId, op);
}
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
@@ -425,7 +429,8 @@
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
}
- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate, runToken });
+ dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,
+ fenceOnly: preparation, runToken });
}
const onMessage = (msg, _sender, respond) => {
if (disposed || msg?.target !== 'content') return;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 81f4ed089..0daf49ea0 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -35169,6 +35169,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
kind: /type|field|key/.test(name) ? 'input' : name === 'scroll' ? 'scroll' : /click|checked|hover|drag/.test(name) ? 'click' : 'dom',
frameId: messageOptions?.frameId || 0, selector: contentArgs?.selector, ref_id: contentArgs?.ref_id,
navigationCandidate: ['click', 'click_ax', 'set_checked'].includes(name),
+ fenceOnly: true,
});
}
throwIfContentPipelineAborted();
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index 318afaaad..8eaa79faf 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -147,6 +147,7 @@ export class FirefoxBidiClient {
...(payload.point ? { x: payload.point.x, y: payload.point.y } : {}),
selector: payload.selector, ref_id: payload.ref_id, frameId: payload.frameId || 0,
navigationCandidate: ['click', 'checked'].includes(action),
+ fenceOnly: true,
url: action === 'navigate' ? payload.url : undefined,
});
if (!owner.bound) {
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index c5fc8beed..80442de92 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -188,7 +188,11 @@
for (const record of records) {
if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;
if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;
- for (const el of [...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100))
+ const descendants = new Set([
+ ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),
+ ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),
+ ]);
+ for (const el of descendants)
changes.push({ type: 'layout', target: el });
}
let measured = 0;
@@ -391,7 +395,7 @@
}
op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);
op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);
- op.until = Date.now() + 1500; op.dispatched = true;
+ op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;
operations.set(op.operationId, op);
}
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
@@ -425,7 +429,8 @@
const marker = { userAt: lastUserAt }; agentTurn = marker;
setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
}
- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate, runToken });
+ dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,
+ fenceOnly: preparation, runToken });
}
const onMessage = (msg, _sender, respond) => {
if (disposed || msg?.target !== 'content') return;
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 5dc2eaac8..0273edc6b 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -22,6 +22,7 @@ test('uploads register input dispatches with the extension monitor', async () =>
assert.equal(registrations.length, 1);
assert.equal(registrations[0].kind, 'input');
assert.equal(registrations[0].navigationCandidate, false);
+ assert.equal(registrations[0].fenceOnly, true);
} finally { owner._finishPageFeedbackRun(tab); }
});
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 5808d67b7..f5de6bdd9 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -138,6 +138,53 @@ for (const siteIsolation of [false, true]) {
});
}
+test('Firefox native preparation keeps same-target user edits and sends no stale keys', async () => {
+ const { FirefoxBidiClient } = await import('../src/firefox/src/bidi/client.js');
+ const { pageFeedbackMethods: methods } = await import('../src/firefox/src/agent/page-feedback.js');
+ const { browser, page } = await fixture(firefox, 'firefox');
+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser;
+ const tab = 960, runId = crypto.randomUUID(), token = crypto.randomUUID(), session = new BidiSession();
+ const host = { ...methods, isRunning: () => true, _checkAbort: () => false };
+ let releasePreparation, startPreparation, inputs = 0;
+ const entered = new Promise(resolve => { startPreparation = resolve; });
+ const release = new Promise(resolve => { releasePreparation = resolve; });
+ const api = { tabs: { get: async () => ({ url: page.url() }), sendMessage: async (_tab, message) => page.evaluate(message => {
+ deliver(message.action, message.params || {}); return lastMonitorResponse;
+ }, message) } };
+ globalThis.chrome = api; globalThis.browser = api;
+ const client = new FirefoxBidiClient(api);
+ session.runs.set(runId, { context: 'tab' });
+ session.locate = async () => { startPreparation(); await release; return { context: 'tab', node: { sharedId: 'field' } }; };
+ session.call = async (_match, declaration, args = []) => ({ result: { value: await page.evaluate(({ declaration, args }) =>
+ (0, eval)(`(${declaration})`)(document.getElementById('field'), ...args.map(arg => arg.value)), { declaration, args }) } });
+ session.send = async method => { if (method === 'input.performActions') inputs++; return {}; };
+ client.runs.set(tab, { runId, bound: true });
+ client.request = async (_command, args) => session.perform(args.runId, args.action, args.payload);
+ try {
+ await host._beginPageFeedbackRun(tab, 'interactive');
+ const runToken = host._pageFeedbackRuns.get(tab).token;
+ await page.evaluate(runToken => { monitorRunToken = runToken; monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state'); }, runToken);
+ await page.waitForTimeout(200);
+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);
+ const from = { tab: { id: tab }, frameId: 0 };
+ host.pageMonitorState(from, documentToken);
+ await page.evaluate(token => document.getElementById('field').setAttribute('data-webbrain-bidi', token), token);
+ await page.locator('#field').focus();
+ const action = client.perform(tab, 'type', { token, selector: '#field', text: 'agent', clear: false });
+ await entered;
+ await page.keyboard.insertText('human');
+ for (const event of await page.evaluate(() => feedback)) host.observePageFeedback(from, event);
+ releasePreparation();
+ await assert.rejects(action, error => error.code === 'page_feedback_pending' && error.dispatchState.noDispatch === true);
+ assert.equal(inputs, 0);
+ assert.equal(host._hasPendingPageFeedback(tab), true);
+ assert.equal(await page.locator('#field').inputValue(), 'human');
+ } finally {
+ releasePreparation(); await session.close(); host._finishPageFeedbackRun(tab); await browser.close();
+ globalThis.chrome = savedChrome; globalThis.browser = savedBrowser;
+ }
+});
+
for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
test(`${build}: monitor captures user input without values, suppresses agent input and keeps concurrent user input`, async () => {
const { browser, page } = await fixture(engine, build);
@@ -337,7 +384,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
try {
const guard = await page.evaluate(() => {
deliver('page_monitor_prepare', { operationId: 'native-typing', tool: 'type_text', selector: '#field' });
- deliver('page_monitor_dispatch', { operationId: 'native-typing', kind: 'input', selector: '#field', navigationCandidate: false });
+ deliver('page_monitor_dispatch', { operationId: 'native-typing', kind: 'input', selector: '#field', navigationCandidate: false, fenceOnly: true });
feedback = []; return lastMonitorResponse.guard;
});
await page.locator('#field').focus();
@@ -355,6 +402,66 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ for (const action of ['click', 'type']) {
+ test(`${build}: ${action} preparation preserves same-target human input`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const selector = action === 'click' ? '#agent' : '#field';
+ await page.evaluate(({ action, selector }) => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'preparing', tool: action, selector });
+ deliver('page_monitor_dispatch', { operationId: 'preparing', kind: action === 'click' ? 'click' : 'input', selector, fenceOnly: true });
+ window.finishPreparation = __wbPageMonitor.beginContentAction(action, { selector, _bidiPrepare: action === 'type' });
+ }, { action, selector });
+ if (action === 'click') await page.locator(selector).click();
+ else { await page.locator(selector).focus(); await page.keyboard.insertText('human'); }
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'),
+ 'Preparation must not consume physical input as an agent action');
+ const code = await page.evaluate(() => {
+ try { __wbPageMonitor.beforeLocalDispatch(); return null; }
+ catch (error) { return error.code; }
+ finally { finishPreparation(); }
+ });
+ assert.equal(code, 'page_feedback_pending');
+ } finally { await browser.close(); }
+ });
+ }
+
+ test(`${build}: ancestor class and style changes track plain content visibility`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const style = document.createElement('style'); style.textContent = '#plain-result {display:var(--result-display,block)} .closed #plain-result {display:none}'; document.head.append(style);
+ const container = document.createElement('div'); container.id = 'style-container'; container.className = 'closed';
+ container.style.cssText = 'width:200px;height:80px'; container.innerHTML = '<div id="plain-result">Visible result content</div>';
+ document.body.prepend(container);
+ });
+ for (const attribute of ['class', 'style']) {
+ await page.evaluate(attribute => {
+ const container = document.getElementById('style-container');
+ container.className = 'closed'; container.style.setProperty('--result-display', attribute === 'style' ? 'none' : 'block');
+ if (attribute === 'style') container.className = '';
+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');
+ }, attribute);
+ await page.waitForTimeout(200);
+ for (const show of [true, false]) {
+ const unchangedGeometry = await page.evaluate(({ attribute, show }) => {
+ feedback = [];
+ const container = document.getElementById('style-container'), before = container.getBoundingClientRect();
+ if (attribute === 'class') container.className = show ? '' : 'closed';
+ else container.style.setProperty('--result-display', show ? 'block' : 'none');
+ const after = container.getBoundingClientRect();
+ return before.width === after.width && before.height === after.height;
+ }, { attribute, show });
+ assert.equal(unchangedGeometry, true);
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.equal(await page.locator('#plain-result').isVisible(), show);
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom'));
+ }
+ }
+ } finally { await browser.close(); }
+ });
+
test(`${build}: same-shape subtree replacements invalidate prepared actions`, async () => {
const { browser, page } = await fixture(engine, build);
try {
@@ -444,7 +551,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
await page.waitForTimeout(200);
const guard = await page.evaluate(() => {
deliver('page_monitor_prepare', { operationId: 'clear-chord', tool: 'set_field', selector: '#field' });
- deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field' });
+ deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field', fenceOnly: true });
feedback = []; return lastMonitorResponse.guard;
});
const mark = sequence => page.evaluate(({ guard, sequence }) => document.getElementById('field')
@@ -482,7 +589,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
await page.locator('#field').focus();
const guard = await page.evaluate(() => {
deliver('page_monitor_prepare', { operationId: 'upload', tool: 'upload_file', selector: '#upload' });
- deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false });
+ deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false, fenceOnly: true });
feedback = []; return lastMonitorResponse.guard;
});
assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);