Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: defer input attribution until actual dispatch
Barack Sokullu9615a2a2fed5
diff --git a/docs/page-feedback-monitor.md b/docs/page-feedback-monitor.mdindex 31692cb59..a3cea5d26 100644--- a/docs/page-feedback-monitor.md+++ b/docs/page-feedback-monitor.md@@ -45,6 +45,9 @@ Steer or grant authorization. Enter and Alt+Enter retain their existing behaviorresults have matching messages. In-flight results remain; undispatched siblingcalls receive explicit skipped results. Transport and local dispatch gatescover interventions during asynchronous preparation.+- Content and Firefox native preparation handshakes check the revision without+ claiming input events. Local dispatch and native markers activate expectations+ at the mutation boundary, preserving same-target intervention during preparation.- Both Chrome upload paths check feedback after preparation, immediately beforeattaching files. A skipped attachment retains its undispatched outcome.- Firefox companion uploads register as input and place their native marker on@@ -86,4 +89,9 @@ container scrolling, animation noise, cleanup/restart and persisted navigationnotes. Browser extension APIs are mocked in these fixtures; they are not a livecompanion-session integration test.+Visibility regressions include fixed-size ancestors changing class or CSS custom+properties to reveal or hide non-interactive content. Native preparation fixtures+also exercise the real companion code across an asynchronous target lookup while+the user edits the same field.+The monitor regressions are also included in `npm run test:runtime-lifecycle`.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 713a2ef16..9504debb1 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -40698,6 +40698,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaikind: /type|field|key/.test(name) ? 'input' : name === 'scroll' ? 'scroll' : /click|checked|hover|drag/.test(name) ? 'click' : 'dom',frameId: messageOptions?.frameId || 0, selector: contentArgs?.selector, ref_id: contentArgs?.ref_id,navigationCandidate: ['click', 'click_ax', 'set_checked'].includes(name),+ fenceOnly: true,});}if (earlyCdpAbortSignal) {diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex c5fc8beed..80442de92 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -188,7 +188,11 @@for (const record of records) {if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;- for (const el of [...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100))+ const descendants = new Set([+ ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),+ ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),+ ]);+ for (const el of descendants)changes.push({ type: 'layout', target: el });}let measured = 0;@@ -391,7 +395,7 @@}op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);- op.until = Date.now() + 1500; op.dispatched = true;+ op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;operations.set(op.operationId, op);}const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',@@ -425,7 +429,8 @@const marker = { userAt: lastUserAt }; agentTurn = marker;setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);}- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate, runToken });+ dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,+ fenceOnly: preparation, runToken });}const onMessage = (msg, _sender, respond) => {if (disposed || msg?.target !== 'content') return;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 81f4ed089..0daf49ea0 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -35169,6 +35169,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dkind: /type|field|key/.test(name) ? 'input' : name === 'scroll' ? 'scroll' : /click|checked|hover|drag/.test(name) ? 'click' : 'dom',frameId: messageOptions?.frameId || 0, selector: contentArgs?.selector, ref_id: contentArgs?.ref_id,navigationCandidate: ['click', 'click_ax', 'set_checked'].includes(name),+ fenceOnly: true,});}throwIfContentPipelineAborted();diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex 318afaaad..8eaa79faf 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -147,6 +147,7 @@ export class FirefoxBidiClient {...(payload.point ? { x: payload.point.x, y: payload.point.y } : {}),selector: payload.selector, ref_id: payload.ref_id, frameId: payload.frameId || 0,navigationCandidate: ['click', 'checked'].includes(action),+ fenceOnly: true,url: action === 'navigate' ? payload.url : undefined,});if (!owner.bound) {diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex c5fc8beed..80442de92 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -188,7 +188,11 @@for (const record of records) {if (record.type !== 'attributes' || !['class', 'style', 'hidden', 'aria-hidden'].includes(record.attributeName)) continue;if (record.target.getAnimations?.().some(animation => animation.playState === 'running')) continue;- for (const el of [...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100))+ const descendants = new Set([+ ...[...(record.target.querySelectorAll?.('button,a,input,textarea,select,[role],[contenteditable]') || [])].slice(0, 100),+ ...[...(record.target.querySelectorAll?.('*') || [])].slice(0, 100),+ ]);+ for (const el of descendants)changes.push({ type: 'layout', target: el });}let measured = 0;@@ -391,7 +395,7 @@}op.windowScroll = params.scrollIntoView || (params.kind === 'scroll' && !op.target) || ['input', 'click'].includes(params.kind);op.kinds = new Set([params.kind, 'dom', ...(params.kind === 'input' ? ['selection'] : []), 'scroll']);- op.until = Date.now() + 1500; op.dispatched = true;+ op.until = Date.now() + (params.fenceOnly ? 30000 : 1500); op.dispatched = !params.fenceOnly;operations.set(op.operationId, op);}const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',@@ -425,7 +429,8 @@const marker = { userAt: lastUserAt }; agentTurn = marker;setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);}- dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate, runToken });+ dispatch({ ...localOperation, navigationCandidate: !preparation && localOperation.navigationCandidate,+ fenceOnly: preparation, runToken });}const onMessage = (msg, _sender, respond) => {if (disposed || msg?.target !== 'content') return;diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 5dc2eaac8..0273edc6b 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -22,6 +22,7 @@ test('uploads register input dispatches with the extension monitor', async () =>assert.equal(registrations.length, 1);assert.equal(registrations[0].kind, 'input');assert.equal(registrations[0].navigationCandidate, false);+ assert.equal(registrations[0].fenceOnly, true);} finally { owner._finishPageFeedbackRun(tab); }});diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 5808d67b7..f5de6bdd9 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -138,6 +138,53 @@ for (const siteIsolation of [false, true]) {});}+test('Firefox native preparation keeps same-target user edits and sends no stale keys', async () => {+ const { FirefoxBidiClient } = await import('../src/firefox/src/bidi/client.js');+ const { pageFeedbackMethods: methods } = await import('../src/firefox/src/agent/page-feedback.js');+ const { browser, page } = await fixture(firefox, 'firefox');+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser;+ const tab = 960, runId = crypto.randomUUID(), token = crypto.randomUUID(), session = new BidiSession();+ const host = { ...methods, isRunning: () => true, _checkAbort: () => false };+ let releasePreparation, startPreparation, inputs = 0;+ const entered = new Promise(resolve => { startPreparation = resolve; });+ const release = new Promise(resolve => { releasePreparation = resolve; });+ const api = { tabs: { get: async () => ({ url: page.url() }), sendMessage: async (_tab, message) => page.evaluate(message => {+ deliver(message.action, message.params || {}); return lastMonitorResponse;+ }, message) } };+ globalThis.chrome = api; globalThis.browser = api;+ const client = new FirefoxBidiClient(api);+ session.runs.set(runId, { context: 'tab' });+ session.locate = async () => { startPreparation(); await release; return { context: 'tab', node: { sharedId: 'field' } }; };+ session.call = async (_match, declaration, args = []) => ({ result: { value: await page.evaluate(({ declaration, args }) =>+ (0, eval)(`(${declaration})`)(document.getElementById('field'), ...args.map(arg => arg.value)), { declaration, args }) } });+ session.send = async method => { if (method === 'input.performActions') inputs++; return {}; };+ client.runs.set(tab, { runId, bound: true });+ client.request = async (_command, args) => session.perform(args.runId, args.action, args.payload);+ try {+ await host._beginPageFeedbackRun(tab, 'interactive');+ const runToken = host._pageFeedbackRuns.get(tab).token;+ await page.evaluate(runToken => { monitorRunToken = runToken; monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state'); }, runToken);+ await page.waitForTimeout(200);+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);+ const from = { tab: { id: tab }, frameId: 0 };+ host.pageMonitorState(from, documentToken);+ await page.evaluate(token => document.getElementById('field').setAttribute('data-webbrain-bidi', token), token);+ await page.locator('#field').focus();+ const action = client.perform(tab, 'type', { token, selector: '#field', text: 'agent', clear: false });+ await entered;+ await page.keyboard.insertText('human');+ for (const event of await page.evaluate(() => feedback)) host.observePageFeedback(from, event);+ releasePreparation();+ await assert.rejects(action, error => error.code === 'page_feedback_pending' && error.dispatchState.noDispatch === true);+ assert.equal(inputs, 0);+ assert.equal(host._hasPendingPageFeedback(tab), true);+ assert.equal(await page.locator('#field').inputValue(), 'human');+ } finally {+ releasePreparation(); await session.close(); host._finishPageFeedbackRun(tab); await browser.close();+ globalThis.chrome = savedChrome; globalThis.browser = savedBrowser;+ }+});+for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {test(`${build}: monitor captures user input without values, suppresses agent input and keeps concurrent user input`, async () => {const { browser, page } = await fixture(engine, build);@@ -337,7 +384,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {try {const guard = await page.evaluate(() => {deliver('page_monitor_prepare', { operationId: 'native-typing', tool: 'type_text', selector: '#field' });- deliver('page_monitor_dispatch', { operationId: 'native-typing', kind: 'input', selector: '#field', navigationCandidate: false });+ deliver('page_monitor_dispatch', { operationId: 'native-typing', kind: 'input', selector: '#field', navigationCandidate: false, fenceOnly: true });feedback = []; return lastMonitorResponse.guard;});await page.locator('#field').focus();@@ -355,6 +402,66 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ for (const action of ['click', 'type']) {+ test(`${build}: ${action} preparation preserves same-target human input`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const selector = action === 'click' ? '#agent' : '#field';+ await page.evaluate(({ action, selector }) => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'preparing', tool: action, selector });+ deliver('page_monitor_dispatch', { operationId: 'preparing', kind: action === 'click' ? 'click' : 'input', selector, fenceOnly: true });+ window.finishPreparation = __wbPageMonitor.beginContentAction(action, { selector, _bidiPrepare: action === 'type' });+ }, { action, selector });+ if (action === 'click') await page.locator(selector).click();+ else { await page.locator(selector).focus(); await page.keyboard.insertText('human'); }+ assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'),+ 'Preparation must not consume physical input as an agent action');+ const code = await page.evaluate(() => {+ try { __wbPageMonitor.beforeLocalDispatch(); return null; }+ catch (error) { return error.code; }+ finally { finishPreparation(); }+ });+ assert.equal(code, 'page_feedback_pending');+ } finally { await browser.close(); }+ });+ }++ test(`${build}: ancestor class and style changes track plain content visibility`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const style = document.createElement('style'); style.textContent = '#plain-result {display:var(--result-display,block)} .closed #plain-result {display:none}'; document.head.append(style);+ const container = document.createElement('div'); container.id = 'style-container'; container.className = 'closed';+ container.style.cssText = 'width:200px;height:80px'; container.innerHTML = '<div id="plain-result">Visible result content</div>';+ document.body.prepend(container);+ });+ for (const attribute of ['class', 'style']) {+ await page.evaluate(attribute => {+ const container = document.getElementById('style-container');+ container.className = 'closed'; container.style.setProperty('--result-display', attribute === 'style' ? 'none' : 'block');+ if (attribute === 'style') container.className = '';+ monitorEnabled = false; deliver('page_monitor_state'); monitorEnabled = true; deliver('page_monitor_state');+ }, attribute);+ await page.waitForTimeout(200);+ for (const show of [true, false]) {+ const unchangedGeometry = await page.evaluate(({ attribute, show }) => {+ feedback = [];+ const container = document.getElementById('style-container'), before = container.getBoundingClientRect();+ if (attribute === 'class') container.className = show ? '' : 'closed';+ else container.style.setProperty('--result-display', show ? 'block' : 'none');+ const after = container.getBoundingClientRect();+ return before.width === after.width && before.height === after.height;+ }, { attribute, show });+ assert.equal(unchangedGeometry, true);+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.equal(await page.locator('#plain-result').isVisible(), show);+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom'));+ }+ }+ } finally { await browser.close(); }+ });+test(`${build}: same-shape subtree replacements invalidate prepared actions`, async () => {const { browser, page } = await fixture(engine, build);try {@@ -444,7 +551,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await page.waitForTimeout(200);const guard = await page.evaluate(() => {deliver('page_monitor_prepare', { operationId: 'clear-chord', tool: 'set_field', selector: '#field' });- deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field' });+ deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field', fenceOnly: true });feedback = []; return lastMonitorResponse.guard;});const mark = sequence => page.evaluate(({ guard, sequence }) => document.getElementById('field')@@ -482,7 +589,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await page.locator('#field').focus();const guard = await page.evaluate(() => {deliver('page_monitor_prepare', { operationId: 'upload', tool: 'upload_file', selector: '#upload' });- deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false });+ deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false, fenceOnly: true });feedback = []; return lastMonitorResponse.guard;});assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);