Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix remaining page feedback monitor review findings
Barack Sokullu9d5bb6d6822c
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 0b00685fa..c10657104 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -38718,6 +38718,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai},},);+ if ((injected?.noDispatch === true || injected?.dispatched === false) && injected?.outcomeUnknown !== true) {+ uploadDispatched = false;+ earlyCdpDispatchState.started = false;+ }if (injected?.deadlineExpired && injected?.dispatched !== true) {uploadDispatched = false;earlyCdpDispatchState.started = false;@@ -38736,6 +38740,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaireturn {success: false,dispatched: injected?.dispatched === true,+ ...(injected?.noDispatch === true || injected?.dispatched === false ? { noDispatch: true } : {}),+ ...(injected?.pageFeedbackPending === true ? { pageFeedbackPending: true } : {}),error: `Upload failed: ${injected?.error || 'the page rejected the attached file data'}`,};}diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex eefe4074c..4663d6171 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -19,12 +19,12 @@ const SELECTOR_SCROLL_DISPATCH_SOURCE = ``;const PAGE_AGENT_DOM_ACTION_SOURCE = `- const beforePageAgentDomAction = (guard, phase) => {+ const beforePageAgentDomAction = (guard, phase, target = window) => {if (!guard) return true;const event = new CustomEvent('webbrain-agent-dom-dispatch', {- detail: JSON.stringify({ ...guard, dispatchPhase: phase }), cancelable: true,+ detail: JSON.stringify({ ...guard, dispatchPhase: phase }), bubbles: true, composed: true, cancelable: true,});- return window.dispatchEvent(event);+ return target.dispatchEvent(event);};`;@@ -2590,11 +2590,14 @@ export class CDPClient {await this.sendCommand(tabId, 'Runtime.enable');throwIfAborted();const target = hasPageAgentDispatchOwner(tabId) ? await this._pageAgentObjectTarget(tabId, objectId) : {};- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'input', eventTypes: ['input', 'change'], ...target });+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, {+ kind: 'input', eventTypes: ['input', 'change'], fenceOnly: true, prepareMonitor: true, ...target,+ });throwIfAborted();if (typeof options?.beforeDispatch === 'function') options.beforeDispatch();const res = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {- functionDeclaration: `function (base64, filename, mimeType, actionDeadlineAt) {+ functionDeclaration: `function (base64, filename, mimeType, actionDeadlineAt, pageGuard) {+ ${PAGE_AGENT_DOM_ACTION_SOURCE}const deadlineExpired = () => Number(actionDeadlineAt) > 0 && Date.now() >= Number(actionDeadlineAt);if (deadlineExpired()) {return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };@@ -2613,6 +2616,10 @@ export class CDPClient {if (deadlineExpired()) {return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };}+ if (!beforePageAgentDomAction(pageGuard, 'input', this)) {+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,+ error: 'The page changed before the in-memory upload could dispatch.' };+ }this.files = transfer.files;dispatched = true;this.dispatchEvent(new Event('input', { bubbles: true }));@@ -2629,6 +2636,7 @@ export class CDPClient {{ value: String(filename || 'attachment') },{ value: String(mimeType || 'application/octet-stream') },{ value: deadlineAt },+ { value: pageGuard || null },],returnByValue: true,});diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 688066ccf..60edf28e2 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -14,6 +14,7 @@const documentToken = randomToken();const listeners = [];const operations = new Map();+ const agentLayoutHistory = [];const nativeTargets = new Set();const liveControls = new Set();let signatures = new WeakMap();@@ -170,13 +171,7 @@if (related(target, el) || related(el, target)) return true;// Editing a target can move a following sibling. Associate only the// matching displacement from a recent, expected input on that target.- for (const cause of op.layoutCauses || []) {- if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;- const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;- if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)- || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;- return true;- }+ if ((op.layoutCauses || []).some(cause => layoutCauseMatches(cause, el, shift))) return true;// Scroll operations may change layout inside their actual scroll box.// Ignore document-wide ancestors so an unrelated page shift is not claimed.for (const region of op.scrollAncestors || []) {@@ -195,14 +190,32 @@if (layoutAgentOperation(op, el, shift)) return op.userAt;}for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;+ pruneAgentLayoutHistory();+ const cause = agentLayoutHistory.find(entry => lastUserAt <= entry.userAt && layoutCauseMatches(entry, el, shift));+ if (cause) return cause.userAt;return undefined;}+ function pruneAgentLayoutHistory() {+ const now = Date.now();+ for (let i = agentLayoutHistory.length - 1; i >= 0; i--) if (agentLayoutHistory[i].until < now) agentLayoutHistory.splice(i, 1);+ }+ function layoutCauseMatches(cause, el, shift) {+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) return false;+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;+ return Math.abs(sourceTopDelta) > 2 && Math.sign(sourceTopDelta) === Math.sign(cause.heightDelta)+ && Math.abs(sourceTopDelta) <= Math.abs(cause.heightDelta) + 8;+ }function rememberAgentLayout(op) {if (!op?.target) return;+ pruneAgentLayoutHistory();const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);if (before && after && Math.abs(after.height - before.height) > 2) {const causes = op.layoutCauses ||= [];- causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });+ const cause = { parent: op.target.parentElement, heightDelta: after.height - before.height,+ userAt: op.userAt, until: Date.now() + 3000 };+ causes.push(cause);+ agentLayoutHistory.push(cause);+ if (agentLayoutHistory.length > 256) agentLayoutHistory.splice(0, agentLayoutHistory.length - 256);if (causes.length > 8) causes.splice(0, causes.length - 8);}if (after) op.layoutRect = after;@@ -348,6 +361,12 @@trackControl(el);controlSignatures.set(el, controlState(el));}+ if (el.tagName === 'SELECT') {+ for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {+ controlSignatures.set(option, controlState(option));+ signatures.set(option, signature(option));+ }+ }signatures.set(el, signature(el));}function signature(el) {@@ -495,6 +514,21 @@return false;}));}+ function hasVisibleAriaLabelledbyConsumer(el) {+ const ids = new Set();+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {+ if (node.id) ids.add(node.id);+ }+ if (!ids.size) return false;+ let consumers;+ try { consumers = document.querySelectorAll('[aria-labelledby]'); } catch { return false; }+ for (const consumer of consumers) {+ if (ignored(consumer) || !visible(consumer)) continue;+ const references = (consumer.getAttribute('aria-labelledby') || '').trim().split(/\s+/);+ if (references.some(id => ids.has(id))) return true;+ }+ return false;+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -539,12 +573,13 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;- const editableCharacterData = record.type === 'characterData' && editable(el);+ const editableTextMutation = editable(el) && (record.type === 'characterData'+ || (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));if (ignored(el)) continue;// Native input already reports user edits, and agent input's synchronous// DOM writes are attributed to its dispatch. Keep those paths deduplicated// while observing independent page-script changes to editable text.- if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;+ if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;if (++measured > 300) {@@ -563,12 +598,15 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}+ const hiddenAccessibleNameChanged = !visible(el)+ && ['characterData', 'childList'].includes(record.type)+ && hasVisibleAriaLabelledbyConsumer(el);let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)- || editableCharacterData+ || editableTextMutation || hiddenAccessibleNameChanged|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {- identityChanged = subtreeChanged(record, el);+ identityChanged ||= subtreeChanged(record, el);for (const node of record.addedNodes) {if (node.nodeType !== 1 || ignored(node)) continue;if (seedBudget.remaining > 0) { seedBudget.remaining--; signatures.set(node, signature(node)); }@@ -641,7 +679,8 @@try {const guard = JSON.parse(String(event.detail));const op = operations.get(guard.operationId);- const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const phase = ['focus', 'click', 'input'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const phaseKind = phase === 'input' ? 'input' : phase === 'dom' ? 'dom' : 'click';const actionPhase = phase !== 'dom';const phaseAlreadyConsumed = phase === 'dom'? op?.domDispatchConsumed === true@@ -650,17 +689,19 @@flushPendingMutations();sampleFormControls(op?.target);if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision- || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')+ || !op || phaseAlreadyConsumed || op.kind !== phaseKind+ || (phase === 'input' && elementFor(event) !== op.target && elementFor(event) !== op.focusTarget)|| op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {event.preventDefault(); return;}if (actionPhase) {op.domDispatchPhases ||= new Set();op.domDispatchPhases.add(phase);- if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: phaseKind, runToken });+ if (phase === 'input') op.synchronous = true;} else op.domDispatchConsumed = true;const marker = { userAt: lastUserAt }; agentTurn = marker;- setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);+ setTimeout(() => { if (phase === 'input') op.synchronous = false; if (agentTurn === marker) agentTurn = null; }, 0);} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }}, false);listen(document, 'webbrain-agent-scroll-dispatch', event => {@@ -815,6 +856,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;+ agentLayoutHistory.length = 0;roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();}async function requestState() {@@ -911,7 +953,12 @@const previous = op.synchronous;op.synchronous = true;try { return callback(); }- finally { op.synchronous = previous; }+ finally {+ op.synchronous = previous;+ // Native editing can move following siblings; record the causal size+ // change now, before the browser reports its layout shift asynchronously.+ rememberAgentLayout(op);+ }}const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 688066ccf..60edf28e2 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -14,6 +14,7 @@const documentToken = randomToken();const listeners = [];const operations = new Map();+ const agentLayoutHistory = [];const nativeTargets = new Set();const liveControls = new Set();let signatures = new WeakMap();@@ -170,13 +171,7 @@if (related(target, el) || related(el, target)) return true;// Editing a target can move a following sibling. Associate only the// matching displacement from a recent, expected input on that target.- for (const cause of op.layoutCauses || []) {- if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;- const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;- if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)- || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;- return true;- }+ if ((op.layoutCauses || []).some(cause => layoutCauseMatches(cause, el, shift))) return true;// Scroll operations may change layout inside their actual scroll box.// Ignore document-wide ancestors so an unrelated page shift is not claimed.for (const region of op.scrollAncestors || []) {@@ -195,14 +190,32 @@if (layoutAgentOperation(op, el, shift)) return op.userAt;}for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;+ pruneAgentLayoutHistory();+ const cause = agentLayoutHistory.find(entry => lastUserAt <= entry.userAt && layoutCauseMatches(entry, el, shift));+ if (cause) return cause.userAt;return undefined;}+ function pruneAgentLayoutHistory() {+ const now = Date.now();+ for (let i = agentLayoutHistory.length - 1; i >= 0; i--) if (agentLayoutHistory[i].until < now) agentLayoutHistory.splice(i, 1);+ }+ function layoutCauseMatches(cause, el, shift) {+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) return false;+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;+ return Math.abs(sourceTopDelta) > 2 && Math.sign(sourceTopDelta) === Math.sign(cause.heightDelta)+ && Math.abs(sourceTopDelta) <= Math.abs(cause.heightDelta) + 8;+ }function rememberAgentLayout(op) {if (!op?.target) return;+ pruneAgentLayoutHistory();const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);if (before && after && Math.abs(after.height - before.height) > 2) {const causes = op.layoutCauses ||= [];- causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });+ const cause = { parent: op.target.parentElement, heightDelta: after.height - before.height,+ userAt: op.userAt, until: Date.now() + 3000 };+ causes.push(cause);+ agentLayoutHistory.push(cause);+ if (agentLayoutHistory.length > 256) agentLayoutHistory.splice(0, agentLayoutHistory.length - 256);if (causes.length > 8) causes.splice(0, causes.length - 8);}if (after) op.layoutRect = after;@@ -348,6 +361,12 @@trackControl(el);controlSignatures.set(el, controlState(el));}+ if (el.tagName === 'SELECT') {+ for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {+ controlSignatures.set(option, controlState(option));+ signatures.set(option, signature(option));+ }+ }signatures.set(el, signature(el));}function signature(el) {@@ -495,6 +514,21 @@return false;}));}+ function hasVisibleAriaLabelledbyConsumer(el) {+ const ids = new Set();+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {+ if (node.id) ids.add(node.id);+ }+ if (!ids.size) return false;+ let consumers;+ try { consumers = document.querySelectorAll('[aria-labelledby]'); } catch { return false; }+ for (const consumer of consumers) {+ if (ignored(consumer) || !visible(consumer)) continue;+ const references = (consumer.getAttribute('aria-labelledby') || '').trim().split(/\s+/);+ if (references.some(id => ids.has(id))) return true;+ }+ return false;+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -539,12 +573,13 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;- const editableCharacterData = record.type === 'characterData' && editable(el);+ const editableTextMutation = editable(el) && (record.type === 'characterData'+ || (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));if (ignored(el)) continue;// Native input already reports user edits, and agent input's synchronous// DOM writes are attributed to its dispatch. Keep those paths deduplicated// while observing independent page-script changes to editable text.- if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;+ if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;if (++measured > 300) {@@ -563,12 +598,15 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}+ const hiddenAccessibleNameChanged = !visible(el)+ && ['characterData', 'childList'].includes(record.type)+ && hasVisibleAriaLabelledbyConsumer(el);let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)- || editableCharacterData+ || editableTextMutation || hiddenAccessibleNameChanged|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));if (record.type === 'childList') {- identityChanged = subtreeChanged(record, el);+ identityChanged ||= subtreeChanged(record, el);for (const node of record.addedNodes) {if (node.nodeType !== 1 || ignored(node)) continue;if (seedBudget.remaining > 0) { seedBudget.remaining--; signatures.set(node, signature(node)); }@@ -641,7 +679,8 @@try {const guard = JSON.parse(String(event.detail));const op = operations.get(guard.operationId);- const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const phase = ['focus', 'click', 'input'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';+ const phaseKind = phase === 'input' ? 'input' : phase === 'dom' ? 'dom' : 'click';const actionPhase = phase !== 'dom';const phaseAlreadyConsumed = phase === 'dom'? op?.domDispatchConsumed === true@@ -650,17 +689,19 @@flushPendingMutations();sampleFormControls(op?.target);if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision- || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')+ || !op || phaseAlreadyConsumed || op.kind !== phaseKind+ || (phase === 'input' && elementFor(event) !== op.target && elementFor(event) !== op.focusTarget)|| op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {event.preventDefault(); return;}if (actionPhase) {op.domDispatchPhases ||= new Set();op.domDispatchPhases.add(phase);- if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: phaseKind, runToken });+ if (phase === 'input') op.synchronous = true;} else op.domDispatchConsumed = true;const marker = { userAt: lastUserAt }; agentTurn = marker;- setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);+ setTimeout(() => { if (phase === 'input') op.synchronous = false; if (agentTurn === marker) agentTurn = null; }, 0);} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }}, false);listen(document, 'webbrain-agent-scroll-dispatch', event => {@@ -815,6 +856,7 @@clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;+ agentLayoutHistory.length = 0;roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();}async function requestState() {@@ -911,7 +953,12 @@const previous = op.synchronous;op.synchronous = true;try { return callback(); }- finally { op.synchronous = previous; }+ finally {+ op.synchronous = previous;+ // Native editing can move following siblings; record the causal size+ // change now, before the browser reports its layout shift asynchronously.+ rememberAgentLayout(op);+ }}const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 1e24b1f5a..e6a354d38 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -896,21 +896,27 @@ for (const path of ['local', 'memory']) {const originalMessage = api.tabs.sendMessage;try {const binding = bind(agent, tab), child = bind(agent, tab, 2, 'upload-doc', 'upload-token');- const client = new CDPClient();+ const client = new CDPClient(), monitorGuards = [], runtimeCalls = [];client._pageAgentObjectTarget = async () => {if (intervention === 'preparation') binding.send({ kind: 'click' });return { documentToken: child.state.documentToken, documentRevision: 0, nativeTarget: 'upload-target' };};api.tabs.sendMessage = async (_tab, message, options) => {+ if (message.action === 'page_monitor_prepare') return { ready: true };if (message.action === 'page_monitor_dispatch') {assert.equal(options.frameId, 2);notices.push(message);if (intervention === 'handshake') binding.send({ kind: 'click' });+ const guard = { runToken: message.params.runToken, documentToken: child.state.documentToken,+ revision: 0, operationId: message.params.operationId };+ monitorGuards.push(guard);+ return { ready: true, guard };}return {};};client.sendCommand = async (_tab, method, params) => {if (method === 'DOM.setFileInputFiles' || method === 'Runtime.callFunctionOn') mutations.push(method);+ if (method === 'Runtime.callFunctionOn') runtimeCalls.push(params);return { result: { value: { success: true, dispatched: true } } };};let started = false;@@ -922,6 +928,12 @@ for (const path of ['local', 'memory']) {assert.equal(started, true);assert.equal(mutations.length, 1);assert.equal(notices.length, 1);+ if (path === 'memory') {+ const call = runtimeCalls[0];+ assert.deepEqual(call.arguments.at(-1), { value: monitorGuards[0] });+ assert.ok(call.functionDeclaration.indexOf("beforePageAgentDomAction(pageGuard, 'input', this)")+ < call.functionDeclaration.indexOf('this.files = transfer.files'));+ }} else {await assert.rejects(upload, { code: 'page_feedback_pending' });assert.equal(started, false);diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 2065b709a..33305ab46 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1095,6 +1095,19 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: native select input does not emit a delayed duplicate option change`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => { feedback = []; });+ await page.locator('#select').selectOption('B');+ await page.waitForFunction(() => feedback.some(event => event.kind === 'input'), null, { timeout: 1000 });+ await page.waitForTimeout(350);+ const observed = await page.evaluate(() => feedback);+ assert.equal(observed.some(event => event.kind === 'dom' && event.target === 'option'), false,+ 'Sampling the option property after its user input must not replan the run a second time');+ } finally { await browser.close(); }+ });+test(`${build}: native markers reject replay, tampering and copied targets`, async () => {const { browser, page } = await fixture(engine, build);try {@@ -1389,7 +1402,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {if (behavior === 'editor-handles') { event.preventDefault(); editor.append(' handled'); }};}, behavior);- await page.waitForTimeout(200);+ await page.waitForTimeout(400);await page.evaluate(() => { feedback = []; });const result = await page.evaluate(params => typeRichText(params), {text: behavior === 'replace' ? 'New\ntext' : ' added', clear: behavior === 'replace',@@ -1405,8 +1418,8 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {if (behavior === 'append') assert.equal(await page.evaluate(() => document.getElementById('rich-editor').contains(originalStrong)), true);else assert.equal(result.value, 'New\ntext');}- await page.waitForTimeout(200);- assert.deepEqual(await page.evaluate(() => feedback), [], 'Own beforeinput, input and DOM effects must not trigger replanning');+ await page.waitForTimeout(400);+ assert.deepEqual(await page.evaluate(() => feedback), [], `Own beforeinput, input and DOM effects must not trigger replanning (${behavior})`);}await page.evaluate(() => {const editor = document.getElementById('rich-editor'); editor.onbeforeinput = null; editor.textContent = 'Ready';@@ -1591,6 +1604,112 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {assert.ok(observations.some(event => event.source === 'page' && event.target.startsWith('div#programmatic-editor')));assert.equal(JSON.stringify(observations).includes('Page updated this editor'), false,'Editable content must invalidate state without being included in feedback');++ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'editable-replacement', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'editable-replacement', kind: 'click', selector: '#agent', fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ await page.evaluate(() => { document.getElementById('programmatic-editor').textContent = 'Page replaced the editor value'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ const replacementObservations = await page.evaluate(() => feedback);+ assert.ok(replacementObservations.some(event => event.source === 'page'+ && event.target.startsWith('div#programmatic-editor')),+ 'Replacing an editable text node must advance the page revision');+ assert.equal(JSON.stringify(replacementObservations).includes('Page replaced the editor value'), false,+ 'Replaced editable text must remain private');+ const ready = await page.evaluate(async () => {+ deliver('page_monitor_dispatch', { operationId: 'editable-replacement', kind: 'click', selector: '#agent' });+ await new Promise(resolve => setTimeout(resolve, 0));+ return lastMonitorResponse;+ });+ assert.equal(ready.pageFeedbackPending, true, 'A prepared action must not survive editable text replacement');+ assert.ok(replacementObservations.some(observation => observation.revision > guard.revision));+ } finally { await browser.close(); }+ });++ test(`${build}: hidden aria-labelledby text changes invalidate visible targets`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const label = document.createElement('span'); label.id = 'hidden-accessible-label'; label.textContent = 'Initial label';+ label.style.display = 'none';+ const button = document.createElement('button'); button.id = 'labelled-button'; button.setAttribute('aria-labelledby', label.id);+ document.body.append(label, button);+ });+ await page.waitForTimeout(180);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'hidden-label', tool: 'click', selector: '#labelled-button' });+ deliver('page_monitor_dispatch', { operationId: 'hidden-label', kind: 'click', selector: '#labelled-button', fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ await page.evaluate(() => { document.getElementById('hidden-accessible-label').textContent = 'Updated label'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ const observations = await page.evaluate(() => feedback);+ assert.ok(observations.some(event => event.source === 'page'));+ assert.equal(JSON.stringify(observations).includes('Updated label'), false, 'Hidden accessible-name text must not be sent');+ const accepted = await page.evaluate(value => {+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), bubbles: true, composed: true, cancelable: true,+ });+ return document.getElementById('labelled-button').dispatchEvent(event);+ }, guard);+ assert.equal(accepted, false, 'Changing a hidden accessible label must invalidate the prepared click');+ } finally { await browser.close(); }+ });++ test(`${build}: guarded synthetic file input events are attributed once`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const input = document.createElement('input'); input.id = 'synthetic-upload'; input.type = 'file'; document.body.append(input);+ });+ await page.waitForTimeout(200);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'synthetic-upload', tool: 'upload_file', selector: '#synthetic-upload' });+ deliver('page_monitor_dispatch', { operationId: 'synthetic-upload', kind: 'input', selector: '#synthetic-upload',+ eventTypes: ['input', 'change'], fenceOnly: true });+ return lastMonitorResponse.guard;+ });+ const accepted = await page.locator('#synthetic-upload').evaluate((input, value) => {+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'input' }), bubbles: true, composed: true, cancelable: true,+ });+ if (!input.dispatchEvent(event)) return false;+ const transfer = new DataTransfer(); transfer.items.add(new File(['fixture'], 'fixture.txt', { type: 'text/plain' }));+ input.files = transfer.files;+ input.dispatchEvent(new Event('input', { bubbles: true }));+ input.dispatchEvent(new Event('change', { bubbles: true }));+ return true;+ }, guard);+ assert.equal(accepted, true);+ await page.waitForTimeout(200);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'The guarded synthetic upload must not interrupt its own run');++ const staleGuard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'stale-synthetic-upload', tool: 'upload_file', selector: '#synthetic-upload' });+ deliver('page_monitor_dispatch', { operationId: 'stale-synthetic-upload', kind: 'input', selector: '#synthetic-upload',+ eventTypes: ['input', 'change'], fenceOnly: true });+ const value = lastMonitorResponse.guard;+ document.getElementById('status').textContent = 'Page changed before upload dispatch';+ return value;+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ const staleAccepted = await page.locator('#synthetic-upload').evaluate((input, value) => {+ let inputEvents = 0;+ input.addEventListener('input', () => inputEvents++, { once: true });+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'input' }), bubbles: true, composed: true, cancelable: true,+ });+ const okay = input.dispatchEvent(event);+ if (okay) input.dispatchEvent(new Event('input', { bubbles: true }));+ return { okay, inputEvents };+ }, staleGuard);+ assert.deepEqual(staleAccepted, { okay: false, inputEvents: 0 }, 'Stale upload guards must stop the synthetic input sequence');} finally { await browser.close(); }});