Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix page feedback dispatch fencing
Barack Sokullu265d0786096a
diff --git a/firefox-companion/host.mjs b/firefox-companion/host.mjsindex da8823dba..41e683958 100644--- a/firefox-companion/host.mjs+++ b/firefox-companion/host.mjs@@ -3,6 +3,8 @@ import { BidiSession } from './session.mjs';import { nativeReplyMessages } from './native-messages.mjs';const session = new BidiSession();let buffer = Buffer.alloc(0);+const pendingPageDispatchChecks = new Map();+let pageDispatchCheckSequence = 0;function reply(value) {for (const message of nativeReplyMessages(value)) {const data = Buffer.from(JSON.stringify(message));@@ -10,7 +12,26 @@ function reply(value) {process.stdout.write(Buffer.concat([header, data]));}}+function validatePageDispatch(runId, guard, kind, rebindFocus = false) {+ const id = `page-dispatch-${++pageDispatchCheckSequence}`;+ return new Promise(resolve => {+ const timer = setTimeout(() => {+ pendingPageDispatchChecks.delete(id);+ resolve(false);+ }, 5000);+ pendingPageDispatchChecks.set(id, result => {+ clearTimeout(timer);+ pendingPageDispatchChecks.delete(id);+ resolve(result === true);+ });+ reply({ id, command: 'validatePageDispatch', runId, guard, kind, rebindFocus: rebindFocus === true });+ });+}async function dispatch(message) {+ if (typeof message?.replyTo === 'string') {+ pendingPageDispatchChecks.get(message.replyTo)?.(message.result);+ return;+ }const { id, command, ...args } = message;try {let result;@@ -18,7 +39,7 @@ async function dispatch(message) {case 'connect': result = await session.connect(args.port); break;case 'openRun': result = await session.openRun(args.runId, args.token, args.url); break;case 'closeRun': result = await session.closeRun(args.runId); break;- case 'perform': result = await session.perform(args.runId, args.action, args.payload || {}); break;+ case 'perform': result = await session.perform(args.runId, args.action, args.payload || {}, validatePageDispatch); break;case 'captureFullPage': result = await session.captureFullPage(args.token, args.url); break;default: throw new Error('Unknown companion command');}@@ -35,4 +56,7 @@ process.stdin.on('data', data => {try { void dispatch(JSON.parse(data)); } catch { process.exit(1); }}});-process.stdin.on('end', () => { void session.close().finally(() => process.exit()); });+process.stdin.on('end', () => {+ for (const settle of pendingPageDispatchChecks.values()) settle(false);+ void session.close().finally(() => process.exit());+});diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex c7bd3f2a0..e93028c5d 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -21,8 +21,8 @@ function pageActionSignature(secret, data) {v0 = (v0 + v3) & sipMask; v3 = sipRotate(v3, 21); v3 ^= v0;v2 = (v2 + v1) & sipMask; v1 = sipRotate(v1, 17); v1 ^= v2; v2 = sipRotate(v2, 32);};- const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,- data.navigationCandidate === true]));+ const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,+ data.kind, data.sequence, data.navigationCandidate === true]));let offset = 0;while (offset + 8 <= bytes.length) {let block = 0n;@@ -245,9 +245,9 @@ export class BidiSession {if (run) await this.send('input.releaseActions', { context: run.context }).catch(() => {});return {};}- async perform(id, action, payload) {+ async perform(id, action, payload, validatePageDispatch = null) {const dispatch = { started: false };- try { return await this.performAction(id, action, payload, dispatch); }+ try { return await this.performAction(id, action, payload, dispatch, validatePageDispatch); }catch (error) {error.dispatchState = { dispatched: dispatch.started, noDispatch: !dispatch.started,outcomeUnknown: dispatch.started, retryable: !dispatch.started,@@ -255,7 +255,7 @@ export class BidiSession {throw error;}}- async performAction(id, action, payload, dispatch) {+ async performAction(id, action, payload, dispatch, validatePageDispatch = null) {if (!['navigate', 'click', 'hover', 'type', 'field', 'key', 'upload'].includes(action)) throw new Error('Unsupported BiDi action');if (payload.modifiers) throw new Error('Key modifiers are not supported by this tool');const run = this.runs.get(id); if (!run) throw new Error('Run stopped or disconnected');@@ -274,18 +274,22 @@ export class BidiSession {const match = await this.locate(payload.token, payload.url, run.context);assertLive();let nativeSequence = 0;- const markNativeDispatch = async kind => {+ const markNativeDispatch = async (kind, { rebindFocus = false } = {}) => {assertLive();const guard = payload.pageFeedbackGuard;if (!guard) return; // Compatibility with clients without page monitoring.+ if (typeof validatePageDispatch !== 'function'+ || !await validatePageDispatch(id, guard, kind, rebindFocus === true)) {+ const error = new Error('Page changed during native preparation; no further input sent');+ error.code = 'page_feedback_pending'; throw error;+ }const marker = createNativeActionMarker(guard, kind, ++nativeSequence);- const checked = await this.call(match, `(el, fence, marker, kind, action) => {- if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;+ const checked = await this.call(match, `(el, marker, kind, action) => {+ if (!el.isConnected) return false;const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;target.setAttribute('data-webbrain-native-action', marker);return true;- }`, [{ type: 'string', value: `${guard.documentToken}:${guard.revision}` },- { type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);+ }`, [{ type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);if (checked.result?.value !== true) {const error = new Error('Page changed during native preparation; no further input sent');error.code = 'page_feedback_pending'; throw error;@@ -390,7 +394,7 @@ export class BidiSession {} finally { await clearNativeDispatch(marker); }await assertFocus();}- const marker = await markNativeDispatch('input');+ const marker = await markNativeDispatch('input', { rebindFocus: followTabFocus });try {dispatch.started = true;await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex aa5659780..1bd194dbb 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -38212,7 +38212,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiabortSignal: earlyCdpAbortSignal,deadlineAt: Number(CONTENT_ACTION_SIGNAL_DEADLINES.get(earlyCdpAbortSignal)?.deadlineAt) || 0,deadlineError: CONTENT_ACTION_SIGNAL_DEADLINES.get(earlyCdpAbortSignal)?.error || null,- beforeDispatch: () => {+ beforeDispatch: async frameId => {+ await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'input', selector, frameId, fenceOnly: true,+ });dispatched = true;markEarlyCdpDispatched();},@@ -38224,6 +38227,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaireturn legacyResult;}throwIfEarlyCdpAborted();+ await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,+ });dispatched = true;markEarlyCdpDispatched();try {diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex af5bb0ae1..1a65b3f22 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -82,6 +82,23 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {return acknowledgement?.guard;}+/** Validate a BiDi native action against the isolated monitor's private revision at dispatch time. */+export async function validateNativePageDispatch(api, tabId, guard, { kind = 'input', rebindFocus = false } = {}) {+ const owner = dispatchOwners.get(tabId);+ if (!owner || owner.runToken !== guard?.runToken || owner.operationId !== guard?.operationId || owner.pending()) return false;+ const frameId = owner.frameForDocument(guard.documentToken);+ if (!Number.isSafeInteger(frameId) || frameId < 0) return false;+ owner.operationFrames.get(owner.operationId)?.add(frameId);+ let acknowledgement;+ try {+ acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_validate', params: {+ runToken: guard.runToken, documentToken: guard.documentToken, operationId: guard.operationId,+ revision: guard.revision, kind, rebindFocus: rebindFocus === true,+ } }, { frameId });+ } catch { return false; }+ return dispatchOwners.get(tabId) === owner && !owner.pending() && acknowledgement?.ready === true;+}+function safeAction(tool, args = {}) {const target = {};for (const key of ['selector', 'ref_id', 'textMatch', 'urlFilter']) {diff --git a/src/chrome/src/agent/rich-text-toolbar-probe.js b/src/chrome/src/agent/rich-text-toolbar-probe.jsindex ce0af7e18..9f39c9348 100644--- a/src/chrome/src/agent/rich-text-toolbar-probe.js+++ b/src/chrome/src/agent/rich-text-toolbar-probe.js@@ -291,7 +291,7 @@ export class RichTextToolbarProbe {}markerPrepared = true;throwIfAborted();- if (typeof beforeDispatch === 'function') beforeDispatch();+ if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);const results = await chrome.scripting.executeScript({target: { tabId, frameIds: [selected.frameId] },func: (markerAttribute, markerValue, txt, clr, actionDeadlineAt) => {diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 0715e26f3..91b5912c5 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -52,8 +52,8 @@v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);};- const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,- data.navigationCandidate === true]));+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,+ data.kind, data.sequence, data.navigationCandidate === true]));let offset = 0;while (offset + 8 <= bytes.length) {let block = 0n;@@ -244,9 +244,11 @@const allowedEvent = data.kind === 'input'? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type): data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision+ && !domTimer && !unreported && lastUserAt <= op.userAt;const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence- && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {+ && fresh && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {// The capability is bound to the isolated-world prepared node and consumed once.op.nativeSequence = data.sequence;markerNode.removeAttribute(nativeMarkerAttribute);@@ -354,6 +356,8 @@return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),+ el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),+ el.getAttribute('formaction'), el.getAttribute('formtarget'),el.getAttribute('popover'), popoverOpen(el),el.selected === true, el.defaultSelected === true,Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,@@ -412,6 +416,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',+ 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {const el = elementFor(event);@@ -825,6 +830,21 @@publishRevision();return { ready: true, operationId: op.operationId, revision };}+ function validateNativeDispatch(params = {}) {+ sampleFormControls();+ const op = operations.get(params.operationId);+ const shifted = coordinatePreparationShifted(op);+ if (!active || params.runToken !== runToken || params.documentToken !== documentToken || !op || shifted+ || Number(params.revision) !== op.preparedRevision || op.preparedRevision !== revision+ || domTimer || unreported || lastUserAt > op.userAt) return false;+ if (params.rebindFocus === true) {+ const target = deepActiveElement();+ if (!(op.nativeSequence > 0) || !target || target === document.body || target === document.documentElement) return false;+ op.target = target;+ op.focusTarget = target;+ }+ return true;+ }function withPreparedDispatch(operationId, callback) {const op = operations.get(operationId);if (!op?.dispatched || typeof callback !== 'function') return callback?.();@@ -890,6 +910,7 @@if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }+ else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};if (!params.release) sampleFormControls();diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 15838c504..949b8bb1d 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -35265,7 +35265,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dabortSignal: contentPipelineAbortSignal,deadlineAt: Number(CONTENT_ACTION_SIGNAL_DEADLINES.get(contentPipelineAbortSignal)?.deadlineAt) || 0,deadlineError: CONTENT_ACTION_SIGNAL_DEADLINES.get(contentPipelineAbortSignal)?.error || null,- beforeDispatch: () => {+ beforeDispatch: async frameId => {+ await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {+ kind: 'input', selector, frameId, fenceOnly: true,+ });dispatched = true;markContentPipelineDispatched();},@@ -35277,6 +35280,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn legacyResult;}throwIfContentPipelineAborted();+ await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {+ kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,+ });dispatched = true;markContentPipelineDispatched();try {diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex af5bb0ae1..1a65b3f22 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -82,6 +82,23 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {return acknowledgement?.guard;}+/** Validate a BiDi native action against the isolated monitor's private revision at dispatch time. */+export async function validateNativePageDispatch(api, tabId, guard, { kind = 'input', rebindFocus = false } = {}) {+ const owner = dispatchOwners.get(tabId);+ if (!owner || owner.runToken !== guard?.runToken || owner.operationId !== guard?.operationId || owner.pending()) return false;+ const frameId = owner.frameForDocument(guard.documentToken);+ if (!Number.isSafeInteger(frameId) || frameId < 0) return false;+ owner.operationFrames.get(owner.operationId)?.add(frameId);+ let acknowledgement;+ try {+ acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_validate', params: {+ runToken: guard.runToken, documentToken: guard.documentToken, operationId: guard.operationId,+ revision: guard.revision, kind, rebindFocus: rebindFocus === true,+ } }, { frameId });+ } catch { return false; }+ return dispatchOwners.get(tabId) === owner && !owner.pending() && acknowledgement?.ready === true;+}+function safeAction(tool, args = {}) {const target = {};for (const key of ['selector', 'ref_id', 'textMatch', 'urlFilter']) {diff --git a/src/firefox/src/agent/rich-text-toolbar-probe.js b/src/firefox/src/agent/rich-text-toolbar-probe.jsindex 3991e21d5..1e01751c4 100644--- a/src/firefox/src/agent/rich-text-toolbar-probe.js+++ b/src/firefox/src/agent/rich-text-toolbar-probe.js@@ -299,7 +299,7 @@ export class RichTextToolbarProbe {}markerPrepared = true;throwIfAborted();- if (typeof beforeDispatch === 'function') beforeDispatch();+ if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);const mutationCode = `(() => {let targetDispatched = false;diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex 8eaa79faf..9038c3715 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -1,12 +1,25 @@/** Optional native-messaging transport. No raw BiDi commands are exposed to pages. */-import { beforePageAgentDispatch } from '../agent/page-feedback.js';+import { beforePageAgentDispatch, validateNativePageDispatch } from '../agent/page-feedback.js';export class FirefoxBidiClient {constructor(api) { this.apiOverride = api; this.pending = new Map(); this.runs = new Map(); this.captures = new Map(); this.sequence = 0; this.connectionEpoch = 0; }get api() { return this.apiOverride || globalThis.browser; }request(command, args = {}) {if (!this.port) {this.port = this.api.runtime.connectNative('one.webbrain.bidi');- this.port.onMessage.addListener(message => {+ const port = this.port;+ port.onMessage.addListener(message => {+ if (message?.command === 'validatePageDispatch' && typeof message.id === 'string') {+ void (async () => {+ const entry = [...this.runs.entries()].find(([, run]) => run.runId === message.runId);+ const result = entry ? await validateNativePageDispatch(this.api, entry[0], message.guard, {+ kind: message.kind, rebindFocus: message.rebindFocus === true,+ }) : false;+ try { port.postMessage({ replyTo: message.id, result }); } catch { /* The native host may have disconnected. */ }+ })().catch(() => {+ try { port.postMessage({ replyTo: message.id, result: false }); } catch {}+ });+ return;+ }const entry = this.pending.get(message.id); if (!entry) return;if (typeof message.chunk === 'string') {entry.chunks ||= [];@@ -38,7 +51,6 @@ export class FirefoxBidiClient {entry.reject(error);} else entry.resolve(message.result);});- const port = this.port;this.port.onDisconnect.addListener(() => {if (this.port !== port) return;this.port = null; this.connection = null;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 0715e26f3..91b5912c5 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -52,8 +52,8 @@v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);};- const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,- data.navigationCandidate === true]));+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,+ data.kind, data.sequence, data.navigationCandidate === true]));let offset = 0;while (offset + 8 <= bytes.length) {let block = 0n;@@ -244,9 +244,11 @@const allowedEvent = data.kind === 'input'? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type): data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision+ && !domTimer && !unreported && lastUserAt <= op.userAt;const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence- && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {+ && fresh && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {// The capability is bound to the isolated-world prepared node and consumed once.op.nativeSequence = data.sequence;markerNode.removeAttribute(nativeMarkerAttribute);@@ -354,6 +356,8 @@return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),+ el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),+ el.getAttribute('formaction'), el.getAttribute('formtarget'),el.getAttribute('popover'), popoverOpen(el),el.selected === true, el.defaultSelected === true,Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,@@ -412,6 +416,7 @@roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',+ 'href', 'target', 'download', 'action', 'formaction', 'formtarget','aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });listen(root, 'beforetoggle', event => {const el = elementFor(event);@@ -825,6 +830,21 @@publishRevision();return { ready: true, operationId: op.operationId, revision };}+ function validateNativeDispatch(params = {}) {+ sampleFormControls();+ const op = operations.get(params.operationId);+ const shifted = coordinatePreparationShifted(op);+ if (!active || params.runToken !== runToken || params.documentToken !== documentToken || !op || shifted+ || Number(params.revision) !== op.preparedRevision || op.preparedRevision !== revision+ || domTimer || unreported || lastUserAt > op.userAt) return false;+ if (params.rebindFocus === true) {+ const target = deepActiveElement();+ if (!(op.nativeSequence > 0) || !target || target === document.body || target === document.documentElement) return false;+ op.target = target;+ op.focusTarget = target;+ }+ return true;+ }function withPreparedDispatch(operationId, callback) {const op = operations.get(operationId);if (!op?.dispatched || typeof callback !== 'function') return callback?.();@@ -890,6 +910,7 @@if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }+ else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};if (!params.release) sampleFormControls();diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 0273edc6b..093421b5b 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';import vm from 'node:vm';import { FirefoxBidiClient } from '../src/firefox/src/bidi/client.js';import { BidiSession } from '../firefox-companion/session.mjs';-import { pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';+import { installPageFeedback, pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';const id = () => crypto.randomUUID();@@ -48,7 +48,7 @@ test('an unfocused upload marks the file input instead of the active control', areturn {};};try {- assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard }, async () => true)).success, true);} finally { await session.close(); }});test('connection loss cannot fall through to synthetic input', async () => {@@ -83,6 +83,77 @@ test('stopping during native tab binding prevents late ownership', async () => {await assert.rejects(opening, /stopped/);assert.equal(session.runs.size, 0);});+test('native dispatch validation returns through the extension-owned frame monitor', async () => {+ const originalBrowser = globalThis.browser;+ const tabId = 956, runId = id(), posts = [], monitorChecks = [];+ let onMessage, preparedOperationId, enterAction, releaseAction;+ const entered = new Promise(resolve => { enterAction = resolve; });+ const release = new Promise(resolve => { releaseAction = resolve; });+ const port = {+ onMessage: { addListener(listener) { onMessage = listener; } },+ onDisconnect: { addListener() {} },+ postMessage(message) { posts.push(message); },+ disconnect() {},+ };+ const api = {+ runtime: { connectNative: () => port },+ tabs: {+ get: async () => ({ url: 'https://example.test/' }),+ sendMessage: async (tab, message, options) => {+ if (message.action === 'page_monitor_prepare') preparedOperationId = message.params.operationId;+ if (message.action === 'page_monitor_validate') {+ monitorChecks.push({ tab, frameId: options?.frameId, params: message.params });+ return { ready: true };+ }+ return {};+ },+ },+ webNavigation: { getAllFrames: async () => [{ frameId: 0 }, { frameId: 9 }] },+ };+ class TestAgent {+ static STATE_CHANGE_TOOLS = new Set(['click']);+ isRunning() { return true; }+ _checkAbort() { return false; }+ _hasPendingPageFeedback() { return false; }+ async executeTool() {+ this.pageMonitorState({ tab: { id: tabId }, frameId: 9, documentId: 'document-9', url: 'https://frame.test/' }, 'monitor-document');+ enterAction();+ await release;+ return { success: true, dispatched: true };+ }+ }+ installPageFeedback(TestAgent);+ const agent = new TestAgent();+ globalThis.browser = api;+ const client = new FirefoxBidiClient(api);+ client.runs.set(tabId, { runId, bound: true });+ try {+ await agent._beginPageFeedbackRun(tabId, 'interactive');+ const action = agent.executeTool(tabId, 'click', { selector: '#target' });+ await entered;+ const request = client.request('ready');+ onMessage({ id: 1, result: true });+ assert.equal(await request, true);+ onMessage({ command: 'validatePageDispatch', id: 'private-check', runId,+ guard: { runToken: agent._pageFeedbackRuns.get(tabId).token, documentToken: 'monitor-document', operationId: preparedOperationId, revision: 4 },+ kind: 'input', rebindFocus: false });+ for (let attempt = 0; attempt < 10 && !posts.some(message => message.replyTo === 'private-check'); attempt++)+ await new Promise(resolve => setTimeout(resolve, 0));+ const response = posts.find(message => message.replyTo === 'private-check');+ assert.deepEqual(response, { replyTo: 'private-check', result: true });+ assert.deepEqual(monitorChecks, [{ tab: tabId, frameId: 9, params: {+ runToken: agent._pageFeedbackRuns.get(tabId).token, documentToken: 'monitor-document',+ operationId: preparedOperationId, revision: 4, kind: 'input', rebindFocus: false,+ } }]);+ releaseAction();+ await action;+ } finally {+ releaseAction();+ agent._finishPageFeedbackRun(tabId);+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});test('dialogs are scoped to a live run and navigation permission is single use', () => {const session = new BidiSession(); const replies = [];session.send = async (method, params) => { replies.push({ method, ...params }); };@@ -112,17 +183,15 @@ test('stop during target preparation prevents input dispatch', async () => {assert.equal(sent.includes('input.performActions'), false);});for (const failAt of [1, 2]) {- test(`page revision change before native input ${failAt} preserves dispatch evidence`, async () => {+ test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {const session = new BidiSession(), runId = id();session.runs.set(runId, { context: 'a' });session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });- let marks = 0, inputs = 0;- session.call = async (_match, declaration) => ({ result: { value:- declaration.includes('data-webbrain-page-revision') ? ++marks < failAt- : declaration.includes('el.innerText : el.value') ? '' : true } });+ let checks = 0, inputs = 0;+ session.call = async (_match, declaration) => ({ result: { value: declaration.includes('el.innerText : el.value') ? '' : true } });session.send = async method => { if (method === 'input.performActions') inputs++; return {}; };await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,- pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }), error => {+ pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {assert.equal(error.code, 'page_feedback_pending');assert.equal(error.dispatchState.noDispatch, failAt === 1);assert.equal(error.dispatchState.dispatched, failAt === 2);@@ -137,13 +206,12 @@ for (const platform of ['Win32', 'MacIntel']) {const session = new BidiSession(), runId = id(), sent = [];session.runs.set(runId, { context: 'a' });session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });- let marks = 0;+ let checks = 0;session.call = async (_match, declaration) => ({ result: { value:- declaration.includes('data-webbrain-page-revision') ? (++marks, true)- : declaration.includes('navigator.platform') ? platform+ declaration.includes('navigator.platform') ? platform: declaration.includes('el.innerText : el.value') ? '' : true } });- session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks, actions: params.actions[0].actions }); return {}; };- await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } });+ session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };+ await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });assert.deepEqual(sent.map(item => item.marks), [1, 2, 3, 4]);assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);assert.equal(sent[1].actions[0].value, 'a');@@ -154,14 +222,13 @@ test('human intervention after the clear modifier blocks A and releases held keyconst session = new BidiSession(), runId = id(), sent = [];session.runs.set(runId, { context: 'a' });session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });- let marks = 0;+ let checks = 0;session.call = async (_match, declaration) => ({ result: { value:- declaration.includes('data-webbrain-page-revision') ? ++marks < 2- : declaration.includes('navigator.platform') ? 'Win32'+ declaration.includes('navigator.platform') ? 'Win32': declaration.includes('el.innerText : el.value') ? '' : true } });session.send = async method => { sent.push(method); return {}; };await assert.rejects(session.perform(runId, 'field', { text: 'x',- pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }), error => {+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 2), error => {assert.equal(error.code, 'page_feedback_pending');assert.equal(error.dispatchState.dispatched, true);assert.equal(error.dispatchState.outcomeUnknown, true);@@ -170,6 +237,24 @@ test('human intervention after the clear modifier blocks A and releases held keyassert.equal(sent.filter(method => method === 'input.performActions').length, 1);assert.equal(sent.at(-1), 'input.releaseActions');});+test('repeated native Tab validates and rebinds to the focused target before each follow-up', async () => {+ const session = new BidiSession(), runId = id(), validations = [], sent = [];+ session.runs.set(runId, { context: 'a' });+ session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });+ session.call = async () => ({ result: { value: true } });+ session.send = async (method, params) => { if (method === 'input.performActions') sent.push(params); return {}; };+ const result = await session.perform(runId, 'key', { key: 'Tab', repeat: 3,+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async (_id, _guard, kind, rebindFocus) => {+ validations.push({ kind, rebindFocus }); return true;+ });+ assert.equal(result.success, true);+ assert.deepEqual(validations, [+ { kind: 'input', rebindFocus: false },+ { kind: 'input', rebindFocus: true },+ { kind: 'input', rebindFocus: true },+ ]);+ assert.equal(sent.length, 3);+});test('unsupported commands and modifiers fail before touching the page', async () => {const session = new BidiSession(); const runId = id();session.runs.set(runId, { context: 'a' });diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 09448a66b..d926c110e 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -888,6 +888,17 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await page.waitForTimeout(80);assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,'An attributed native Tab must not steer its own run');+ const rebound = await page.evaluate(guard => {+ deliver('page_monitor_validate', { ...guard, kind: 'input', rebindFocus: true });+ return lastMonitorResponse.ready;+ }, tabGuard);+ assert.equal(rebound, true, 'A validated follow-up Tab should bind to the newly focused control');+ await page.evaluate(marker => document.activeElement.setAttribute('data-webbrain-native-action', marker),+ createNativeActionMarker(tabGuard, 'input', 2));+ await page.keyboard.press('Tab');+ await page.waitForTimeout(80);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,+ 'The second Tab must remain attributed after native focus moves');} finally { await browser.close(); }});@@ -1096,18 +1107,19 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {const tokens = await Promise.all([page.evaluate(() => feedback[0]?.documentToken), child.evaluate(() => feedback[0]?.documentToken)]);assert.ok(tokens[0] && tokens[1] && tokens[0] !== tokens[1]);await child.evaluate(() => {- const link = document.createElement('a'); link.id = 'top-link'; link.href = 'https://monitor.test/destination';- link.textContent = 'Top link'; link.addEventListener('click', event => event.preventDefault()); document.body.append(link);+ for (const [id, target] of [['top-link', '_top'], ['parent-link', '_parent']]) {+ const link = document.createElement('a'); link.id = id; link.href = 'https://monitor.test/destination';+ link.setAttribute('target', target); link.textContent = id; link.addEventListener('click', event => event.preventDefault()); document.body.append(link);+ }});await page.waitForTimeout(200);- for (const target of ['_top', '_parent']) {- await child.evaluate(target => {- document.getElementById('top-link').target = target;- deliver('page_monitor_prepare', { operationId: `link-${target}`, tool: 'click', selector: '#top-link' });- deliver('page_monitor_dispatch', { operationId: `link-${target}`, kind: 'click', selector: '#top-link' });+ for (const [target, id] of [['_top', 'top-link'], ['_parent', 'parent-link']]) {+ await child.evaluate(({ target, id }) => {+ deliver('page_monitor_prepare', { operationId: `link-${target}`, tool: 'click', selector: `#${id}` });+ deliver('page_monitor_dispatch', { operationId: `link-${target}`, kind: 'click', selector: `#${id}` });feedback = [];- }, target);- await child.locator('#top-link').click();+ }, { target, id });+ await child.locator(`#${id}`).click();assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent' && event.navigationTarget === '_top'),`${target} navigation must carry the compatible top-frame target`);}@@ -1427,6 +1439,41 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: native dispatch validation ignores a page-restored revision attribute`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const guard = await page.evaluate(() => {+ deliver('page_monitor_prepare', { operationId: 'private-revision', tool: 'type_text', selector: '#field' });+ deliver('page_monitor_dispatch', { operationId: 'private-revision', kind: 'input', selector: '#field', fenceOnly: true });+ feedback = []; return lastMonitorResponse.guard;+ });+ await page.evaluate(() => { document.getElementById('status').textContent = 'Page changed after native preparation'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ const ready = await page.evaluate(guard => {+ // The page can rewrite its visible marker, but cannot rewind the monitor's+ // private revision used by the extension-side native dispatch check.+ document.documentElement.setAttribute('data-webbrain-page-revision', `${guard.documentToken}:${guard.revision}`);+ deliver('page_monitor_validate', { ...guard, kind: 'input' });+ return lastMonitorResponse.ready;+ }, guard);+ assert.equal(ready, false);+ } finally { await browser.close(); }+ });++ test(`${build}: link destination mutations are observed`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => { const link = document.createElement('a'); link.id = 'destination'; link.href = '/before'; link.textContent = 'Destination'; document.body.append(link); });+ await page.waitForTimeout(120);+ await page.evaluate(() => { feedback = []; document.getElementById('destination').href = '/after'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.target === 'a#destination'));+ await page.evaluate(() => { feedback = []; document.getElementById('destination').setAttribute('target', '_blank'); });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.target === 'a#destination'));+ } finally { await browser.close(); }+ });+for (const action of ['click', 'type']) {test(`${build}: ${action} preparation preserves same-target human input`, async () => {const { browser, page } = await fixture(engine, build);@@ -1767,7 +1814,10 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false, fenceOnly: true });feedback = []; return lastMonitorResponse.guard;});- assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard },+ async (_runId, currentGuard, kind) => page.evaluate(({ currentGuard, kind }) => {+ deliver('page_monitor_validate', { ...currentGuard, kind }); return lastMonitorResponse.ready;+ }, { currentGuard, kind }))).success, true);await page.waitForTimeout(200);assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);assert.equal(await page.evaluate(() => document.activeElement.id), 'field');diff --git a/test/run.js b/test/run.jsindex 494ebfba0..88925d913 100644--- a/test/run.js+++ b/test/run.js@@ -2477,6 +2477,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phase}if (executePhase === 3) {assert.equal(dispatchMarks, 1, `${build}: mutation injection started without a dispatch marker`);+ assert.equal(dispatchedFrameId, 9, `${build}: native preparation fence must target the selected iframe`);return build === 'chrome'? [{ frameId: 9, result: { ok: true, dispatched: true, method: 'native-setter', value: 'typed' } }]: [{ ok: true, dispatched: true, method: 'native-setter', value: 'typed' }];@@ -2495,6 +2496,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phaseconst moduleUrl = pathToFileURL(path.join(ROOT, `src/${build}/src/agent/rich-text-toolbar-probe.js`)).href;const { RichTextToolbarProbe } = await import(`${moduleUrl}?legacy-mutation-boundary=${build}`);let cleanupWatchdog;+ let dispatchedFrameId = null;const result = await Promise.race([new RichTextToolbarProbe({}).legacyIframeTypeAllFrames(77, {selector: '#editor',@@ -2502,7 +2504,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phaseclear: true,urlFilter: 'frame.test',}, {- beforeDispatch: () => { dispatchMarks += 1; },+ beforeDispatch: async frameId => { await Promise.resolve(); dispatchedFrameId = frameId; dispatchMarks += 1; },}),new Promise((_, reject) => {cleanupWatchdog = setTimeout(() => reject(new Error(`${build}: redundant cleanup blocked a completed mutation`)), 500);@@ -2521,7 +2523,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phaseclear: true,urlFilter: 'frame.test',}, {- beforeDispatch: () => { dispatchMarks += 1; },+ beforeDispatch: async frameId => { await Promise.resolve(); dispatchedFrameId = frameId; dispatchMarks += 1; },});assert.equal(expired.success, false, `${build}: expired preparation unexpectedly succeeded`);assert.equal(expired.deadlineExpired, true, `${build}: preparation expiry was not preserved`);@@ -113117,7 +113119,7 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =`${label}: focus-only iframe expiry is misclassified as a text dispatch`,);const preparationResult = legacyType.indexOf(label === 'chrome' ? 'const prepared = preparedResults?.[0]?.result;' : 'const prepared = (await browser.tabs.executeScript');- const backgroundDispatchMarker = legacyType.indexOf("if (typeof beforeDispatch === 'function') beforeDispatch();", preparationResult);+ const backgroundDispatchMarker = legacyType.indexOf("if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);", preparationResult);const mutationDispatch = legacyType.indexOf(label === 'chrome' ? 'const results = await chrome.scripting.executeScript' : 'code: mutationCode', backgroundDispatchMarker);assert.ok(preparationResult >= 0 && backgroundDispatchMarker > preparationResult && mutationDispatch > backgroundDispatchMarker,@@ -113140,10 +113142,14 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =assert.match(source,label === 'chrome'- ? /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: \(\) => \{\s*dispatched = true;\s*markEarlyCdpDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*earlyCdpDispatchState\.started = false;/- : /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: \(\) => \{\s*dispatched = true;\s*markContentPipelineDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*contentPipelineDispatchState\.started = false;/,+ ? /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: async frameId => \{\s*await beforePageAgentDispatch\(globalThis\.chrome, tabId, \{[\s\S]*kind: 'input', selector, frameId, fenceOnly: true[\s\S]*dispatched = true;\s*markEarlyCdpDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*earlyCdpDispatchState\.started = false;/+ : /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: async frameId => \{\s*await beforePageAgentDispatch\(globalThis\.browser \|\| globalThis\.chrome, tabId, \{[\s\S]*kind: 'input', selector, frameId, fenceOnly: true[\s\S]*dispatched = true;\s*markContentPipelineDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*contentPipelineDispatchState\.started = false;/,`${label}: legacy iframe typing keeps an optimistic dispatch marker after page-proven expiry`,);+ assert.match(source, label === 'chrome'+ ? /await beforePageAgentDispatch\(globalThis\.chrome, tabId, \{\s*kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,/g+ : /await beforePageAgentDispatch\(globalThis\.browser \|\| globalThis\.chrome, tabId, \{\s*kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,/g,+ `${label}: bound iframe typing skips the target-frame feedback fence`);const fallbackTypeStart = contentSource.indexOf('async function _typeTextInner(params, actionDeadlineExpired = () => false)');const fallbackTypeEnd = contentSource.indexOf('\n\n /**', fallbackTypeStart + 20);