Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix page feedback dispatch fencing

Barack Sokullu265d0786096a
diff --git a/firefox-companion/host.mjs b/firefox-companion/host.mjs
index da8823dba..41e683958 100644
--- a/firefox-companion/host.mjs
+++ b/firefox-companion/host.mjs
@@ -3,6 +3,8 @@ import { BidiSession } from './session.mjs';
import { nativeReplyMessages } from './native-messages.mjs';
const session = new BidiSession();
let buffer = Buffer.alloc(0);
+const pendingPageDispatchChecks = new Map();
+let pageDispatchCheckSequence = 0;
function reply(value) {
for (const message of nativeReplyMessages(value)) {
const data = Buffer.from(JSON.stringify(message));
@@ -10,7 +12,26 @@ function reply(value) {
process.stdout.write(Buffer.concat([header, data]));
}
}
+function validatePageDispatch(runId, guard, kind, rebindFocus = false) {
+ const id = `page-dispatch-${++pageDispatchCheckSequence}`;
+ return new Promise(resolve => {
+ const timer = setTimeout(() => {
+ pendingPageDispatchChecks.delete(id);
+ resolve(false);
+ }, 5000);
+ pendingPageDispatchChecks.set(id, result => {
+ clearTimeout(timer);
+ pendingPageDispatchChecks.delete(id);
+ resolve(result === true);
+ });
+ reply({ id, command: 'validatePageDispatch', runId, guard, kind, rebindFocus: rebindFocus === true });
+ });
+}
async function dispatch(message) {
+ if (typeof message?.replyTo === 'string') {
+ pendingPageDispatchChecks.get(message.replyTo)?.(message.result);
+ return;
+ }
const { id, command, ...args } = message;
try {
let result;
@@ -18,7 +39,7 @@ async function dispatch(message) {
case 'connect': result = await session.connect(args.port); break;
case 'openRun': result = await session.openRun(args.runId, args.token, args.url); break;
case 'closeRun': result = await session.closeRun(args.runId); break;
- case 'perform': result = await session.perform(args.runId, args.action, args.payload || {}); break;
+ case 'perform': result = await session.perform(args.runId, args.action, args.payload || {}, validatePageDispatch); break;
case 'captureFullPage': result = await session.captureFullPage(args.token, args.url); break;
default: throw new Error('Unknown companion command');
}
@@ -35,4 +56,7 @@ process.stdin.on('data', data => {
try { void dispatch(JSON.parse(data)); } catch { process.exit(1); }
}
});
-process.stdin.on('end', () => { void session.close().finally(() => process.exit()); });
+process.stdin.on('end', () => {
+ for (const settle of pendingPageDispatchChecks.values()) settle(false);
+ void session.close().finally(() => process.exit());
+});
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index c7bd3f2a0..e93028c5d 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -21,8 +21,8 @@ function pageActionSignature(secret, data) {
v0 = (v0 + v3) & sipMask; v3 = sipRotate(v3, 21); v3 ^= v0;
v2 = (v2 + v1) & sipMask; v1 = sipRotate(v1, 17); v1 ^= v2; v2 = sipRotate(v2, 32);
};
- const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
- data.navigationCandidate === true]));
+ const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,
+ data.kind, data.sequence, data.navigationCandidate === true]));
let offset = 0;
while (offset + 8 <= bytes.length) {
let block = 0n;
@@ -245,9 +245,9 @@ export class BidiSession {
if (run) await this.send('input.releaseActions', { context: run.context }).catch(() => {});
return {};
}
- async perform(id, action, payload) {
+ async perform(id, action, payload, validatePageDispatch = null) {
const dispatch = { started: false };
- try { return await this.performAction(id, action, payload, dispatch); }
+ try { return await this.performAction(id, action, payload, dispatch, validatePageDispatch); }
catch (error) {
error.dispatchState = { dispatched: dispatch.started, noDispatch: !dispatch.started,
outcomeUnknown: dispatch.started, retryable: !dispatch.started,
@@ -255,7 +255,7 @@ export class BidiSession {
throw error;
}
}
- async performAction(id, action, payload, dispatch) {
+ async performAction(id, action, payload, dispatch, validatePageDispatch = null) {
if (!['navigate', 'click', 'hover', 'type', 'field', 'key', 'upload'].includes(action)) throw new Error('Unsupported BiDi action');
if (payload.modifiers) throw new Error('Key modifiers are not supported by this tool');
const run = this.runs.get(id); if (!run) throw new Error('Run stopped or disconnected');
@@ -274,18 +274,22 @@ export class BidiSession {
const match = await this.locate(payload.token, payload.url, run.context);
assertLive();
let nativeSequence = 0;
- const markNativeDispatch = async kind => {
+ const markNativeDispatch = async (kind, { rebindFocus = false } = {}) => {
assertLive();
const guard = payload.pageFeedbackGuard;
if (!guard) return; // Compatibility with clients without page monitoring.
+ if (typeof validatePageDispatch !== 'function'
+ || !await validatePageDispatch(id, guard, kind, rebindFocus === true)) {
+ const error = new Error('Page changed during native preparation; no further input sent');
+ error.code = 'page_feedback_pending'; throw error;
+ }
const marker = createNativeActionMarker(guard, kind, ++nativeSequence);
- const checked = await this.call(match, `(el, fence, marker, kind, action) => {
- if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;
+ const checked = await this.call(match, `(el, marker, kind, action) => {
+ if (!el.isConnected) return false;
const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;
target.setAttribute('data-webbrain-native-action', marker);
return true;
- }`, [{ type: 'string', value: `${guard.documentToken}:${guard.revision}` },
- { type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);
+ }`, [{ type: 'string', value: marker }, { type: 'string', value: kind }, { type: 'string', value: action }]);
if (checked.result?.value !== true) {
const error = new Error('Page changed during native preparation; no further input sent');
error.code = 'page_feedback_pending'; throw error;
@@ -390,7 +394,7 @@ export class BidiSession {
} finally { await clearNativeDispatch(marker); }
await assertFocus();
}
- const marker = await markNativeDispatch('input');
+ const marker = await markNativeDispatch('input', { rebindFocus: followTabFocus });
try {
dispatch.started = true;
await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index aa5659780..1bd194dbb 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -38212,7 +38212,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
abortSignal: earlyCdpAbortSignal,
deadlineAt: Number(CONTENT_ACTION_SIGNAL_DEADLINES.get(earlyCdpAbortSignal)?.deadlineAt) || 0,
deadlineError: CONTENT_ACTION_SIGNAL_DEADLINES.get(earlyCdpAbortSignal)?.error || null,
- beforeDispatch: () => {
+ beforeDispatch: async frameId => {
+ await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'input', selector, frameId, fenceOnly: true,
+ });
dispatched = true;
markEarlyCdpDispatched();
},
@@ -38224,6 +38227,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
return legacyResult;
}
throwIfEarlyCdpAborted();
+ await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,
+ });
dispatched = true;
markEarlyCdpDispatched();
try {
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index af5bb0ae1..1a65b3f22 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -82,6 +82,23 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {
return acknowledgement?.guard;
}
+/** Validate a BiDi native action against the isolated monitor's private revision at dispatch time. */
+export async function validateNativePageDispatch(api, tabId, guard, { kind = 'input', rebindFocus = false } = {}) {
+ const owner = dispatchOwners.get(tabId);
+ if (!owner || owner.runToken !== guard?.runToken || owner.operationId !== guard?.operationId || owner.pending()) return false;
+ const frameId = owner.frameForDocument(guard.documentToken);
+ if (!Number.isSafeInteger(frameId) || frameId < 0) return false;
+ owner.operationFrames.get(owner.operationId)?.add(frameId);
+ let acknowledgement;
+ try {
+ acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_validate', params: {
+ runToken: guard.runToken, documentToken: guard.documentToken, operationId: guard.operationId,
+ revision: guard.revision, kind, rebindFocus: rebindFocus === true,
+ } }, { frameId });
+ } catch { return false; }
+ return dispatchOwners.get(tabId) === owner && !owner.pending() && acknowledgement?.ready === true;
+}
+
function safeAction(tool, args = {}) {
const target = {};
for (const key of ['selector', 'ref_id', 'textMatch', 'urlFilter']) {
diff --git a/src/chrome/src/agent/rich-text-toolbar-probe.js b/src/chrome/src/agent/rich-text-toolbar-probe.js
index ce0af7e18..9f39c9348 100644
--- a/src/chrome/src/agent/rich-text-toolbar-probe.js
+++ b/src/chrome/src/agent/rich-text-toolbar-probe.js
@@ -291,7 +291,7 @@ export class RichTextToolbarProbe {
}
markerPrepared = true;
throwIfAborted();
- if (typeof beforeDispatch === 'function') beforeDispatch();
+ if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);
const results = await chrome.scripting.executeScript({
target: { tabId, frameIds: [selected.frameId] },
func: (markerAttribute, markerValue, txt, clr, actionDeadlineAt) => {
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 0715e26f3..91b5912c5 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -52,8 +52,8 @@
v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;
v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);
};
- const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
- data.navigationCandidate === true]));
+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,
+ data.kind, data.sequence, data.navigationCandidate === true]));
let offset = 0;
while (offset + 8 <= bytes.length) {
let block = 0n;
@@ -244,9 +244,11 @@
const allowedEvent = data.kind === 'input'
? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
: data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision
+ && !domTimer && !unreported && lastUserAt <= op.userAt;
const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence
- && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
+ && fresh && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
// The capability is bound to the isolated-world prepared node and consumed once.
op.nativeSequence = data.sequence;
markerNode.removeAttribute(nativeMarkerAttribute);
@@ -354,6 +356,8 @@
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
+ el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),
+ el.getAttribute('formaction'), el.getAttribute('formtarget'),
el.getAttribute('popover'), popoverOpen(el),
el.selected === true, el.defaultSelected === true,
Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
@@ -412,6 +416,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
+ 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
@@ -825,6 +830,21 @@
publishRevision();
return { ready: true, operationId: op.operationId, revision };
}
+ function validateNativeDispatch(params = {}) {
+ sampleFormControls();
+ const op = operations.get(params.operationId);
+ const shifted = coordinatePreparationShifted(op);
+ if (!active || params.runToken !== runToken || params.documentToken !== documentToken || !op || shifted
+ || Number(params.revision) !== op.preparedRevision || op.preparedRevision !== revision
+ || domTimer || unreported || lastUserAt > op.userAt) return false;
+ if (params.rebindFocus === true) {
+ const target = deepActiveElement();
+ if (!(op.nativeSequence > 0) || !target || target === document.body || target === document.documentElement) return false;
+ op.target = target;
+ op.focusTarget = target;
+ }
+ return true;
+ }
function withPreparedDispatch(operationId, callback) {
const op = operations.get(operationId);
if (!op?.dispatched || typeof callback !== 'function') return callback?.();
@@ -890,6 +910,7 @@
if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }
else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
+ else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
if (!params.release) sampleFormControls();
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 15838c504..949b8bb1d 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -35265,7 +35265,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
abortSignal: contentPipelineAbortSignal,
deadlineAt: Number(CONTENT_ACTION_SIGNAL_DEADLINES.get(contentPipelineAbortSignal)?.deadlineAt) || 0,
deadlineError: CONTENT_ACTION_SIGNAL_DEADLINES.get(contentPipelineAbortSignal)?.error || null,
- beforeDispatch: () => {
+ beforeDispatch: async frameId => {
+ await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {
+ kind: 'input', selector, frameId, fenceOnly: true,
+ });
dispatched = true;
markContentPipelineDispatched();
},
@@ -35277,6 +35280,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return legacyResult;
}
throwIfContentPipelineAborted();
+ await beforePageAgentDispatch(globalThis.browser || globalThis.chrome, tabId, {
+ kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,
+ });
dispatched = true;
markContentPipelineDispatched();
try {
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index af5bb0ae1..1a65b3f22 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -82,6 +82,23 @@ export async function beforePageAgentDispatch(api, tabId, details = {}) {
return acknowledgement?.guard;
}
+/** Validate a BiDi native action against the isolated monitor's private revision at dispatch time. */
+export async function validateNativePageDispatch(api, tabId, guard, { kind = 'input', rebindFocus = false } = {}) {
+ const owner = dispatchOwners.get(tabId);
+ if (!owner || owner.runToken !== guard?.runToken || owner.operationId !== guard?.operationId || owner.pending()) return false;
+ const frameId = owner.frameForDocument(guard.documentToken);
+ if (!Number.isSafeInteger(frameId) || frameId < 0) return false;
+ owner.operationFrames.get(owner.operationId)?.add(frameId);
+ let acknowledgement;
+ try {
+ acknowledgement = await api.tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_validate', params: {
+ runToken: guard.runToken, documentToken: guard.documentToken, operationId: guard.operationId,
+ revision: guard.revision, kind, rebindFocus: rebindFocus === true,
+ } }, { frameId });
+ } catch { return false; }
+ return dispatchOwners.get(tabId) === owner && !owner.pending() && acknowledgement?.ready === true;
+}
+
function safeAction(tool, args = {}) {
const target = {};
for (const key of ['selector', 'ref_id', 'textMatch', 'urlFilter']) {
diff --git a/src/firefox/src/agent/rich-text-toolbar-probe.js b/src/firefox/src/agent/rich-text-toolbar-probe.js
index 3991e21d5..1e01751c4 100644
--- a/src/firefox/src/agent/rich-text-toolbar-probe.js
+++ b/src/firefox/src/agent/rich-text-toolbar-probe.js
@@ -299,7 +299,7 @@ export class RichTextToolbarProbe {
}
markerPrepared = true;
throwIfAborted();
- if (typeof beforeDispatch === 'function') beforeDispatch();
+ if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);
const mutationCode = `
(() => {
let targetDispatched = false;
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index 8eaa79faf..9038c3715 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -1,12 +1,25 @@
/** Optional native-messaging transport. No raw BiDi commands are exposed to pages. */
-import { beforePageAgentDispatch } from '../agent/page-feedback.js';
+import { beforePageAgentDispatch, validateNativePageDispatch } from '../agent/page-feedback.js';
export class FirefoxBidiClient {
constructor(api) { this.apiOverride = api; this.pending = new Map(); this.runs = new Map(); this.captures = new Map(); this.sequence = 0; this.connectionEpoch = 0; }
get api() { return this.apiOverride || globalThis.browser; }
request(command, args = {}) {
if (!this.port) {
this.port = this.api.runtime.connectNative('one.webbrain.bidi');
- this.port.onMessage.addListener(message => {
+ const port = this.port;
+ port.onMessage.addListener(message => {
+ if (message?.command === 'validatePageDispatch' && typeof message.id === 'string') {
+ void (async () => {
+ const entry = [...this.runs.entries()].find(([, run]) => run.runId === message.runId);
+ const result = entry ? await validateNativePageDispatch(this.api, entry[0], message.guard, {
+ kind: message.kind, rebindFocus: message.rebindFocus === true,
+ }) : false;
+ try { port.postMessage({ replyTo: message.id, result }); } catch { /* The native host may have disconnected. */ }
+ })().catch(() => {
+ try { port.postMessage({ replyTo: message.id, result: false }); } catch {}
+ });
+ return;
+ }
const entry = this.pending.get(message.id); if (!entry) return;
if (typeof message.chunk === 'string') {
entry.chunks ||= [];
@@ -38,7 +51,6 @@ export class FirefoxBidiClient {
entry.reject(error);
} else entry.resolve(message.result);
});
- const port = this.port;
this.port.onDisconnect.addListener(() => {
if (this.port !== port) return;
this.port = null; this.connection = null;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 0715e26f3..91b5912c5 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -52,8 +52,8 @@
v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;
v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);
};
- const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
- data.navigationCandidate === true]));
+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.revision,
+ data.kind, data.sequence, data.navigationCandidate === true]));
let offset = 0;
while (offset + 8 <= bytes.length) {
let block = 0n;
@@ -244,9 +244,11 @@
const allowedEvent = data.kind === 'input'
? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
: data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ const fresh = op && data.revision === op.preparedRevision && op.preparedRevision === revision
+ && !domTimer && !unreported && lastUserAt <= op.userAt;
const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence
- && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
+ && fresh && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
// The capability is bound to the isolated-world prepared node and consumed once.
op.nativeSequence = data.sequence;
markerNode.removeAttribute(nativeMarkerAttribute);
@@ -354,6 +356,8 @@
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
+ el.getAttribute('href'), el.getAttribute('target'), el.getAttribute('download'), el.getAttribute('action'),
+ el.getAttribute('formaction'), el.getAttribute('formtarget'),
el.getAttribute('popover'), popoverOpen(el),
el.selected === true, el.defaultSelected === true,
Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
@@ -412,6 +416,7 @@
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
attributeFilter: ['role', 'aria-label', 'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
+ 'href', 'target', 'download', 'action', 'formaction', 'formtarget',
'aria-hidden', 'hidden', 'disabled', 'readonly', 'checked', 'selected', 'open', 'popover', 'inert', 'class', 'style'] });
listen(root, 'beforetoggle', event => {
const el = elementFor(event);
@@ -825,6 +830,21 @@
publishRevision();
return { ready: true, operationId: op.operationId, revision };
}
+ function validateNativeDispatch(params = {}) {
+ sampleFormControls();
+ const op = operations.get(params.operationId);
+ const shifted = coordinatePreparationShifted(op);
+ if (!active || params.runToken !== runToken || params.documentToken !== documentToken || !op || shifted
+ || Number(params.revision) !== op.preparedRevision || op.preparedRevision !== revision
+ || domTimer || unreported || lastUserAt > op.userAt) return false;
+ if (params.rebindFocus === true) {
+ const target = deepActiveElement();
+ if (!(op.nativeSequence > 0) || !target || target === document.body || target === document.documentElement) return false;
+ op.target = target;
+ op.focusTarget = target;
+ }
+ return true;
+ }
function withPreparedDispatch(operationId, callback) {
const op = operations.get(operationId);
if (!op?.dispatched || typeof callback !== 'function') return callback?.();
@@ -890,6 +910,7 @@
if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }
else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
+ else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
if (!params.release) sampleFormControls();
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 0273edc6b..093421b5b 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
import vm from 'node:vm';
import { FirefoxBidiClient } from '../src/firefox/src/bidi/client.js';
import { BidiSession } from '../firefox-companion/session.mjs';
-import { pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';
+import { installPageFeedback, pageFeedbackMethods } from '../src/firefox/src/agent/page-feedback.js';
const id = () => crypto.randomUUID();
@@ -48,7 +48,7 @@ test('an unfocused upload marks the file input instead of the active control', a
return {};
};
try {
- assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);
+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard }, async () => true)).success, true);
} finally { await session.close(); }
});
test('connection loss cannot fall through to synthetic input', async () => {
@@ -83,6 +83,77 @@ test('stopping during native tab binding prevents late ownership', async () => {
await assert.rejects(opening, /stopped/);
assert.equal(session.runs.size, 0);
});
+test('native dispatch validation returns through the extension-owned frame monitor', async () => {
+ const originalBrowser = globalThis.browser;
+ const tabId = 956, runId = id(), posts = [], monitorChecks = [];
+ let onMessage, preparedOperationId, enterAction, releaseAction;
+ const entered = new Promise(resolve => { enterAction = resolve; });
+ const release = new Promise(resolve => { releaseAction = resolve; });
+ const port = {
+ onMessage: { addListener(listener) { onMessage = listener; } },
+ onDisconnect: { addListener() {} },
+ postMessage(message) { posts.push(message); },
+ disconnect() {},
+ };
+ const api = {
+ runtime: { connectNative: () => port },
+ tabs: {
+ get: async () => ({ url: 'https://example.test/' }),
+ sendMessage: async (tab, message, options) => {
+ if (message.action === 'page_monitor_prepare') preparedOperationId = message.params.operationId;
+ if (message.action === 'page_monitor_validate') {
+ monitorChecks.push({ tab, frameId: options?.frameId, params: message.params });
+ return { ready: true };
+ }
+ return {};
+ },
+ },
+ webNavigation: { getAllFrames: async () => [{ frameId: 0 }, { frameId: 9 }] },
+ };
+ class TestAgent {
+ static STATE_CHANGE_TOOLS = new Set(['click']);
+ isRunning() { return true; }
+ _checkAbort() { return false; }
+ _hasPendingPageFeedback() { return false; }
+ async executeTool() {
+ this.pageMonitorState({ tab: { id: tabId }, frameId: 9, documentId: 'document-9', url: 'https://frame.test/' }, 'monitor-document');
+ enterAction();
+ await release;
+ return { success: true, dispatched: true };
+ }
+ }
+ installPageFeedback(TestAgent);
+ const agent = new TestAgent();
+ globalThis.browser = api;
+ const client = new FirefoxBidiClient(api);
+ client.runs.set(tabId, { runId, bound: true });
+ try {
+ await agent._beginPageFeedbackRun(tabId, 'interactive');
+ const action = agent.executeTool(tabId, 'click', { selector: '#target' });
+ await entered;
+ const request = client.request('ready');
+ onMessage({ id: 1, result: true });
+ assert.equal(await request, true);
+ onMessage({ command: 'validatePageDispatch', id: 'private-check', runId,
+ guard: { runToken: agent._pageFeedbackRuns.get(tabId).token, documentToken: 'monitor-document', operationId: preparedOperationId, revision: 4 },
+ kind: 'input', rebindFocus: false });
+ for (let attempt = 0; attempt < 10 && !posts.some(message => message.replyTo === 'private-check'); attempt++)
+ await new Promise(resolve => setTimeout(resolve, 0));
+ const response = posts.find(message => message.replyTo === 'private-check');
+ assert.deepEqual(response, { replyTo: 'private-check', result: true });
+ assert.deepEqual(monitorChecks, [{ tab: tabId, frameId: 9, params: {
+ runToken: agent._pageFeedbackRuns.get(tabId).token, documentToken: 'monitor-document',
+ operationId: preparedOperationId, revision: 4, kind: 'input', rebindFocus: false,
+ } }]);
+ releaseAction();
+ await action;
+ } finally {
+ releaseAction();
+ agent._finishPageFeedbackRun(tabId);
+ if (originalBrowser === undefined) delete globalThis.browser;
+ else globalThis.browser = originalBrowser;
+ }
+});
test('dialogs are scoped to a live run and navigation permission is single use', () => {
const session = new BidiSession(); const replies = [];
session.send = async (method, params) => { replies.push({ method, ...params }); };
@@ -112,17 +183,15 @@ test('stop during target preparation prevents input dispatch', async () => {
assert.equal(sent.includes('input.performActions'), false);
});
for (const failAt of [1, 2]) {
- test(`page revision change before native input ${failAt} preserves dispatch evidence`, async () => {
+ test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {
const session = new BidiSession(), runId = id();
session.runs.set(runId, { context: 'a' });
session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });
- let marks = 0, inputs = 0;
- session.call = async (_match, declaration) => ({ result: { value:
- declaration.includes('data-webbrain-page-revision') ? ++marks < failAt
- : declaration.includes('el.innerText : el.value') ? '' : true } });
+ let checks = 0, inputs = 0;
+ session.call = async (_match, declaration) => ({ result: { value: declaration.includes('el.innerText : el.value') ? '' : true } });
session.send = async method => { if (method === 'input.performActions') inputs++; return {}; };
await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,
- pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }), error => {
+ pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {
assert.equal(error.code, 'page_feedback_pending');
assert.equal(error.dispatchState.noDispatch, failAt === 1);
assert.equal(error.dispatchState.dispatched, failAt === 2);
@@ -137,13 +206,12 @@ for (const platform of ['Win32', 'MacIntel']) {
const session = new BidiSession(), runId = id(), sent = [];
session.runs.set(runId, { context: 'a' });
session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });
- let marks = 0;
+ let checks = 0;
session.call = async (_match, declaration) => ({ result: { value:
- declaration.includes('data-webbrain-page-revision') ? (++marks, true)
- : declaration.includes('navigator.platform') ? platform
+ declaration.includes('navigator.platform') ? platform
: declaration.includes('el.innerText : el.value') ? '' : true } });
- session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks, actions: params.actions[0].actions }); return {}; };
- await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } });
+ session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };
+ await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });
assert.deepEqual(sent.map(item => item.marks), [1, 2, 3, 4]);
assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);
assert.equal(sent[1].actions[0].value, 'a');
@@ -154,14 +222,13 @@ test('human intervention after the clear modifier blocks A and releases held key
const session = new BidiSession(), runId = id(), sent = [];
session.runs.set(runId, { context: 'a' });
session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });
- let marks = 0;
+ let checks = 0;
session.call = async (_match, declaration) => ({ result: { value:
- declaration.includes('data-webbrain-page-revision') ? ++marks < 2
- : declaration.includes('navigator.platform') ? 'Win32'
+ declaration.includes('navigator.platform') ? 'Win32'
: declaration.includes('el.innerText : el.value') ? '' : true } });
session.send = async method => { sent.push(method); return {}; };
await assert.rejects(session.perform(runId, 'field', { text: 'x',
- pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }), error => {
+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 2), error => {
assert.equal(error.code, 'page_feedback_pending');
assert.equal(error.dispatchState.dispatched, true);
assert.equal(error.dispatchState.outcomeUnknown, true);
@@ -170,6 +237,24 @@ test('human intervention after the clear modifier blocks A and releases held key
assert.equal(sent.filter(method => method === 'input.performActions').length, 1);
assert.equal(sent.at(-1), 'input.releaseActions');
});
+test('repeated native Tab validates and rebinds to the focused target before each follow-up', async () => {
+ const session = new BidiSession(), runId = id(), validations = [], sent = [];
+ session.runs.set(runId, { context: 'a' });
+ session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });
+ session.call = async () => ({ result: { value: true } });
+ session.send = async (method, params) => { if (method === 'input.performActions') sent.push(params); return {}; };
+ const result = await session.perform(runId, 'key', { key: 'Tab', repeat: 3,
+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async (_id, _guard, kind, rebindFocus) => {
+ validations.push({ kind, rebindFocus }); return true;
+ });
+ assert.equal(result.success, true);
+ assert.deepEqual(validations, [
+ { kind: 'input', rebindFocus: false },
+ { kind: 'input', rebindFocus: true },
+ { kind: 'input', rebindFocus: true },
+ ]);
+ assert.equal(sent.length, 3);
+});
test('unsupported commands and modifiers fail before touching the page', async () => {
const session = new BidiSession(); const runId = id();
session.runs.set(runId, { context: 'a' });
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 09448a66b..d926c110e 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -888,6 +888,17 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
await page.waitForTimeout(80);
assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,
'An attributed native Tab must not steer its own run');
+ const rebound = await page.evaluate(guard => {
+ deliver('page_monitor_validate', { ...guard, kind: 'input', rebindFocus: true });
+ return lastMonitorResponse.ready;
+ }, tabGuard);
+ assert.equal(rebound, true, 'A validated follow-up Tab should bind to the newly focused control');
+ await page.evaluate(marker => document.activeElement.setAttribute('data-webbrain-native-action', marker),
+ createNativeActionMarker(tabGuard, 'input', 2));
+ await page.keyboard.press('Tab');
+ await page.waitForTimeout(80);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,
+ 'The second Tab must remain attributed after native focus moves');
} finally { await browser.close(); }
});
@@ -1096,18 +1107,19 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
const tokens = await Promise.all([page.evaluate(() => feedback[0]?.documentToken), child.evaluate(() => feedback[0]?.documentToken)]);
assert.ok(tokens[0] && tokens[1] && tokens[0] !== tokens[1]);
await child.evaluate(() => {
- const link = document.createElement('a'); link.id = 'top-link'; link.href = 'https://monitor.test/destination';
- link.textContent = 'Top link'; link.addEventListener('click', event => event.preventDefault()); document.body.append(link);
+ for (const [id, target] of [['top-link', '_top'], ['parent-link', '_parent']]) {
+ const link = document.createElement('a'); link.id = id; link.href = 'https://monitor.test/destination';
+ link.setAttribute('target', target); link.textContent = id; link.addEventListener('click', event => event.preventDefault()); document.body.append(link);
+ }
});
await page.waitForTimeout(200);
- for (const target of ['_top', '_parent']) {
- await child.evaluate(target => {
- document.getElementById('top-link').target = target;
- deliver('page_monitor_prepare', { operationId: `link-${target}`, tool: 'click', selector: '#top-link' });
- deliver('page_monitor_dispatch', { operationId: `link-${target}`, kind: 'click', selector: '#top-link' });
+ for (const [target, id] of [['_top', 'top-link'], ['_parent', 'parent-link']]) {
+ await child.evaluate(({ target, id }) => {
+ deliver('page_monitor_prepare', { operationId: `link-${target}`, tool: 'click', selector: `#${id}` });
+ deliver('page_monitor_dispatch', { operationId: `link-${target}`, kind: 'click', selector: `#${id}` });
feedback = [];
- }, target);
- await child.locator('#top-link').click();
+ }, { target, id });
+ await child.locator(`#${id}`).click();
assert.ok((await child.evaluate(() => feedback)).some(event => event.source === 'agent' && event.navigationTarget === '_top'),
`${target} navigation must carry the compatible top-frame target`);
}
@@ -1427,6 +1439,41 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: native dispatch validation ignores a page-restored revision attribute`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const guard = await page.evaluate(() => {
+ deliver('page_monitor_prepare', { operationId: 'private-revision', tool: 'type_text', selector: '#field' });
+ deliver('page_monitor_dispatch', { operationId: 'private-revision', kind: 'input', selector: '#field', fenceOnly: true });
+ feedback = []; return lastMonitorResponse.guard;
+ });
+ await page.evaluate(() => { document.getElementById('status').textContent = 'Page changed after native preparation'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ const ready = await page.evaluate(guard => {
+ // The page can rewrite its visible marker, but cannot rewind the monitor's
+ // private revision used by the extension-side native dispatch check.
+ document.documentElement.setAttribute('data-webbrain-page-revision', `${guard.documentToken}:${guard.revision}`);
+ deliver('page_monitor_validate', { ...guard, kind: 'input' });
+ return lastMonitorResponse.ready;
+ }, guard);
+ assert.equal(ready, false);
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: link destination mutations are observed`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => { const link = document.createElement('a'); link.id = 'destination'; link.href = '/before'; link.textContent = 'Destination'; document.body.append(link); });
+ await page.waitForTimeout(120);
+ await page.evaluate(() => { feedback = []; document.getElementById('destination').href = '/after'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.target === 'a#destination'));
+ await page.evaluate(() => { feedback = []; document.getElementById('destination').setAttribute('target', '_blank'); });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.ok((await page.evaluate(() => feedback)).some(event => event.kind === 'dom' && event.target === 'a#destination'));
+ } finally { await browser.close(); }
+ });
+
for (const action of ['click', 'type']) {
test(`${build}: ${action} preparation preserves same-target human input`, async () => {
const { browser, page } = await fixture(engine, build);
@@ -1767,7 +1814,10 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
deliver('page_monitor_dispatch', { operationId: 'upload', kind: 'input', selector: '#upload', navigationCandidate: false, fenceOnly: true });
feedback = []; return lastMonitorResponse.guard;
});
- assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard })).success, true);
+ assert.equal((await session.perform(runId, 'upload', { token, base64: 'eA==', filename: 'test.txt', pageFeedbackGuard: guard },
+ async (_runId, currentGuard, kind) => page.evaluate(({ currentGuard, kind }) => {
+ deliver('page_monitor_validate', { ...currentGuard, kind }); return lastMonitorResponse.ready;
+ }, { currentGuard, kind }))).success, true);
await page.waitForTimeout(200);
assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);
assert.equal(await page.evaluate(() => document.activeElement.id), 'field');
diff --git a/test/run.js b/test/run.js
index 494ebfba0..88925d913 100644
--- a/test/run.js
+++ b/test/run.js
@@ -2477,6 +2477,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phase
}
if (executePhase === 3) {
assert.equal(dispatchMarks, 1, `${build}: mutation injection started without a dispatch marker`);
+ assert.equal(dispatchedFrameId, 9, `${build}: native preparation fence must target the selected iframe`);
return build === 'chrome'
? [{ frameId: 9, result: { ok: true, dispatched: true, method: 'native-setter', value: 'typed' } }]
: [{ ok: true, dispatched: true, method: 'native-setter', value: 'typed' }];
@@ -2495,6 +2496,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phase
const moduleUrl = pathToFileURL(path.join(ROOT, `src/${build}/src/agent/rich-text-toolbar-probe.js`)).href;
const { RichTextToolbarProbe } = await import(`${moduleUrl}?legacy-mutation-boundary=${build}`);
let cleanupWatchdog;
+ let dispatchedFrameId = null;
const result = await Promise.race([
new RichTextToolbarProbe({}).legacyIframeTypeAllFrames(77, {
selector: '#editor',
@@ -2502,7 +2504,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phase
clear: true,
urlFilter: 'frame.test',
}, {
- beforeDispatch: () => { dispatchMarks += 1; },
+ beforeDispatch: async frameId => { await Promise.resolve(); dispatchedFrameId = frameId; dispatchMarks += 1; },
}),
new Promise((_, reject) => {
cleanupWatchdog = setTimeout(() => reject(new Error(`${build}: redundant cleanup blocked a completed mutation`)), 500);
@@ -2521,7 +2523,7 @@ test('legacy iframe typing marks background dispatch only for its mutation phase
clear: true,
urlFilter: 'frame.test',
}, {
- beforeDispatch: () => { dispatchMarks += 1; },
+ beforeDispatch: async frameId => { await Promise.resolve(); dispatchedFrameId = frameId; dispatchMarks += 1; },
});
assert.equal(expired.success, false, `${build}: expired preparation unexpectedly succeeded`);
assert.equal(expired.deadlineExpired, true, `${build}: preparation expiry was not preserved`);
@@ -113117,7 +113119,7 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =
`${label}: focus-only iframe expiry is misclassified as a text dispatch`,
);
const preparationResult = legacyType.indexOf(label === 'chrome' ? 'const prepared = preparedResults?.[0]?.result;' : 'const prepared = (await browser.tabs.executeScript');
- const backgroundDispatchMarker = legacyType.indexOf("if (typeof beforeDispatch === 'function') beforeDispatch();", preparationResult);
+ const backgroundDispatchMarker = legacyType.indexOf("if (typeof beforeDispatch === 'function') await beforeDispatch(selected.frameId);", preparationResult);
const mutationDispatch = legacyType.indexOf(label === 'chrome' ? 'const results = await chrome.scripting.executeScript' : 'code: mutationCode', backgroundDispatchMarker);
assert.ok(
preparationResult >= 0 && backgroundDispatchMarker > preparationResult && mutationDispatch > backgroundDispatchMarker,
@@ -113140,10 +113142,14 @@ test('content-script actions have a bounded unknown-outcome timeout', async () =
assert.match(
source,
label === 'chrome'
- ? /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: \(\) => \{\s*dispatched = true;\s*markEarlyCdpDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*earlyCdpDispatchState\.started = false;/
- : /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: \(\) => \{\s*dispatched = true;\s*markContentPipelineDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*contentPipelineDispatchState\.started = false;/,
+ ? /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: async frameId => \{\s*await beforePageAgentDispatch\(globalThis\.chrome, tabId, \{[\s\S]*kind: 'input', selector, frameId, fenceOnly: true[\s\S]*dispatched = true;\s*markEarlyCdpDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*earlyCdpDispatchState\.started = false;/
+ : /const legacyResult = await this\._legacyIframeTypeAllFrames[\s\S]*beforeDispatch: async frameId => \{\s*await beforePageAgentDispatch\(globalThis\.browser \|\| globalThis\.chrome, tabId, \{[\s\S]*kind: 'input', selector, frameId, fenceOnly: true[\s\S]*dispatched = true;\s*markContentPipelineDispatched\(\);[\s\S]*legacyResult\?\.dispatched !== true && legacyResult\?\.noDispatch === true[\s\S]*contentPipelineDispatchState\.started = false;/,
`${label}: legacy iframe typing keeps an optimistic dispatch marker after page-proven expiry`,
);
+ assert.match(source, label === 'chrome'
+ ? /await beforePageAgentDispatch\(globalThis\.chrome, tabId, \{\s*kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,/g
+ : /await beforePageAgentDispatch\(globalThis\.browser \|\| globalThis\.chrome, tabId, \{\s*kind: 'input', selector, frameId: targetFrameId, fenceOnly: true,/g,
+ `${label}: bound iframe typing skips the target-frame feedback fence`);
const fallbackTypeStart = contentSource.indexOf('async function _typeTextInner(params, actionDeadlineExpired = () => false)');
const fallbackTypeEnd = contentSource.indexOf('\n\n /**', fallbackTypeStart + 20);