Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix remaining page feedback monitor review findings

Barack Sokullu9d5bb6d6822c
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 0b00685fa..c10657104 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -38718,6 +38718,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
},
},
);
+ if ((injected?.noDispatch === true || injected?.dispatched === false) && injected?.outcomeUnknown !== true) {
+ uploadDispatched = false;
+ earlyCdpDispatchState.started = false;
+ }
if (injected?.deadlineExpired && injected?.dispatched !== true) {
uploadDispatched = false;
earlyCdpDispatchState.started = false;
@@ -38736,6 +38740,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
return {
success: false,
dispatched: injected?.dispatched === true,
+ ...(injected?.noDispatch === true || injected?.dispatched === false ? { noDispatch: true } : {}),
+ ...(injected?.pageFeedbackPending === true ? { pageFeedbackPending: true } : {}),
error: `Upload failed: ${injected?.error || 'the page rejected the attached file data'}`,
};
}
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index eefe4074c..4663d6171 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -19,12 +19,12 @@ const SELECTOR_SCROLL_DISPATCH_SOURCE = `
`;
const PAGE_AGENT_DOM_ACTION_SOURCE = `
- const beforePageAgentDomAction = (guard, phase) => {
+ const beforePageAgentDomAction = (guard, phase, target = window) => {
if (!guard) return true;
const event = new CustomEvent('webbrain-agent-dom-dispatch', {
- detail: JSON.stringify({ ...guard, dispatchPhase: phase }), cancelable: true,
+ detail: JSON.stringify({ ...guard, dispatchPhase: phase }), bubbles: true, composed: true, cancelable: true,
});
- return window.dispatchEvent(event);
+ return target.dispatchEvent(event);
};
`;
@@ -2590,11 +2590,14 @@ export class CDPClient {
await this.sendCommand(tabId, 'Runtime.enable');
throwIfAborted();
const target = hasPageAgentDispatchOwner(tabId) ? await this._pageAgentObjectTarget(tabId, objectId) : {};
- await beforePageAgentDispatch(globalThis.chrome, tabId, { kind: 'input', eventTypes: ['input', 'change'], ...target });
+ const pageGuard = await beforePageAgentDispatch(globalThis.chrome, tabId, {
+ kind: 'input', eventTypes: ['input', 'change'], fenceOnly: true, prepareMonitor: true, ...target,
+ });
throwIfAborted();
if (typeof options?.beforeDispatch === 'function') options.beforeDispatch();
const res = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {
- functionDeclaration: `function (base64, filename, mimeType, actionDeadlineAt) {
+ functionDeclaration: `function (base64, filename, mimeType, actionDeadlineAt, pageGuard) {
+ ${PAGE_AGENT_DOM_ACTION_SOURCE}
const deadlineExpired = () => Number(actionDeadlineAt) > 0 && Date.now() >= Number(actionDeadlineAt);
if (deadlineExpired()) {
return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };
@@ -2613,6 +2616,10 @@ export class CDPClient {
if (deadlineExpired()) {
return { success: false, dispatched: false, deadlineExpired: true, error: 'Upload action deadline expired before dispatch' };
}
+ if (!beforePageAgentDomAction(pageGuard, 'input', this)) {
+ return { success: false, dispatched: false, noDispatch: true, pageFeedbackPending: true,
+ error: 'The page changed before the in-memory upload could dispatch.' };
+ }
this.files = transfer.files;
dispatched = true;
this.dispatchEvent(new Event('input', { bubbles: true }));
@@ -2629,6 +2636,7 @@ export class CDPClient {
{ value: String(filename || 'attachment') },
{ value: String(mimeType || 'application/octet-stream') },
{ value: deadlineAt },
+ { value: pageGuard || null },
],
returnByValue: true,
});
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 688066ccf..60edf28e2 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -14,6 +14,7 @@
const documentToken = randomToken();
const listeners = [];
const operations = new Map();
+ const agentLayoutHistory = [];
const nativeTargets = new Set();
const liveControls = new Set();
let signatures = new WeakMap();
@@ -170,13 +171,7 @@
if (related(target, el) || related(el, target)) return true;
// Editing a target can move a following sibling. Associate only the
// matching displacement from a recent, expected input on that target.
- for (const cause of op.layoutCauses || []) {
- if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;
- const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
- if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)
- || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;
- return true;
- }
+ if ((op.layoutCauses || []).some(cause => layoutCauseMatches(cause, el, shift))) return true;
// Scroll operations may change layout inside their actual scroll box.
// Ignore document-wide ancestors so an unrelated page shift is not claimed.
for (const region of op.scrollAncestors || []) {
@@ -195,14 +190,32 @@
if (layoutAgentOperation(op, el, shift)) return op.userAt;
}
for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;
+ pruneAgentLayoutHistory();
+ const cause = agentLayoutHistory.find(entry => lastUserAt <= entry.userAt && layoutCauseMatches(entry, el, shift));
+ if (cause) return cause.userAt;
return undefined;
}
+ function pruneAgentLayoutHistory() {
+ const now = Date.now();
+ for (let i = agentLayoutHistory.length - 1; i >= 0; i--) if (agentLayoutHistory[i].until < now) agentLayoutHistory.splice(i, 1);
+ }
+ function layoutCauseMatches(cause, el, shift) {
+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) return false;
+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
+ return Math.abs(sourceTopDelta) > 2 && Math.sign(sourceTopDelta) === Math.sign(cause.heightDelta)
+ && Math.abs(sourceTopDelta) <= Math.abs(cause.heightDelta) + 8;
+ }
function rememberAgentLayout(op) {
if (!op?.target) return;
+ pruneAgentLayoutHistory();
const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);
if (before && after && Math.abs(after.height - before.height) > 2) {
const causes = op.layoutCauses ||= [];
- causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });
+ const cause = { parent: op.target.parentElement, heightDelta: after.height - before.height,
+ userAt: op.userAt, until: Date.now() + 3000 };
+ causes.push(cause);
+ agentLayoutHistory.push(cause);
+ if (agentLayoutHistory.length > 256) agentLayoutHistory.splice(0, agentLayoutHistory.length - 256);
if (causes.length > 8) causes.splice(0, causes.length - 8);
}
if (after) op.layoutRect = after;
@@ -348,6 +361,12 @@
trackControl(el);
controlSignatures.set(el, controlState(el));
}
+ if (el.tagName === 'SELECT') {
+ for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {
+ controlSignatures.set(option, controlState(option));
+ signatures.set(option, signature(option));
+ }
+ }
signatures.set(el, signature(el));
}
function signature(el) {
@@ -495,6 +514,21 @@
return false;
}));
}
+ function hasVisibleAriaLabelledbyConsumer(el) {
+ const ids = new Set();
+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (node.id) ids.add(node.id);
+ }
+ if (!ids.size) return false;
+ let consumers;
+ try { consumers = document.querySelectorAll('[aria-labelledby]'); } catch { return false; }
+ for (const consumer of consumers) {
+ if (ignored(consumer) || !visible(consumer)) continue;
+ const references = (consumer.getAttribute('aria-labelledby') || '').trim().split(/\s+/);
+ if (references.some(id => ids.has(id))) return true;
+ }
+ return false;
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -539,12 +573,13 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
- const editableCharacterData = record.type === 'characterData' && editable(el);
+ const editableTextMutation = editable(el) && (record.type === 'characterData'
+ || (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
if (ignored(el)) continue;
// Native input already reports user edits, and agent input's synchronous
// DOM writes are attributed to its dispatch. Keep those paths deduplicated
// while observing independent page-script changes to editable text.
- if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
+ if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
if (++measured > 300) {
@@ -563,12 +598,15 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
+ const hiddenAccessibleNameChanged = !visible(el)
+ && ['characterData', 'childList'].includes(record.type)
+ && hasVisibleAriaLabelledbyConsumer(el);
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
- || editableCharacterData
+ || editableTextMutation || hiddenAccessibleNameChanged
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
- identityChanged = subtreeChanged(record, el);
+ identityChanged ||= subtreeChanged(record, el);
for (const node of record.addedNodes) {
if (node.nodeType !== 1 || ignored(node)) continue;
if (seedBudget.remaining > 0) { seedBudget.remaining--; signatures.set(node, signature(node)); }
@@ -641,7 +679,8 @@
try {
const guard = JSON.parse(String(event.detail));
const op = operations.get(guard.operationId);
- const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const phase = ['focus', 'click', 'input'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const phaseKind = phase === 'input' ? 'input' : phase === 'dom' ? 'dom' : 'click';
const actionPhase = phase !== 'dom';
const phaseAlreadyConsumed = phase === 'dom'
? op?.domDispatchConsumed === true
@@ -650,17 +689,19 @@
flushPendingMutations();
sampleFormControls(op?.target);
if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision
- || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')
+ || !op || phaseAlreadyConsumed || op.kind !== phaseKind
+ || (phase === 'input' && elementFor(event) !== op.target && elementFor(event) !== op.focusTarget)
|| op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
event.preventDefault(); return;
}
if (actionPhase) {
op.domDispatchPhases ||= new Set();
op.domDispatchPhases.add(phase);
- if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });
+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: phaseKind, runToken });
+ if (phase === 'input') op.synchronous = true;
} else op.domDispatchConsumed = true;
const marker = { userAt: lastUserAt }; agentTurn = marker;
- setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
+ setTimeout(() => { if (phase === 'input') op.synchronous = false; if (agentTurn === marker) agentTurn = null; }, 0);
} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }
}, false);
listen(document, 'webbrain-agent-scroll-dispatch', event => {
@@ -815,6 +856,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
+ agentLayoutHistory.length = 0;
roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
}
async function requestState() {
@@ -911,7 +953,12 @@
const previous = op.synchronous;
op.synchronous = true;
try { return callback(); }
- finally { op.synchronous = previous; }
+ finally {
+ op.synchronous = previous;
+ // Native editing can move following siblings; record the causal size
+ // change now, before the browser reports its layout shift asynchronously.
+ rememberAgentLayout(op);
+ }
}
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
'hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 688066ccf..60edf28e2 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -14,6 +14,7 @@
const documentToken = randomToken();
const listeners = [];
const operations = new Map();
+ const agentLayoutHistory = [];
const nativeTargets = new Set();
const liveControls = new Set();
let signatures = new WeakMap();
@@ -170,13 +171,7 @@
if (related(target, el) || related(el, target)) return true;
// Editing a target can move a following sibling. Associate only the
// matching displacement from a recent, expected input on that target.
- for (const cause of op.layoutCauses || []) {
- if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) continue;
- const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
- if (Math.abs(sourceTopDelta) <= 2 || Math.sign(sourceTopDelta) !== Math.sign(cause.heightDelta)
- || Math.abs(sourceTopDelta) > Math.abs(cause.heightDelta) + 8) continue;
- return true;
- }
+ if ((op.layoutCauses || []).some(cause => layoutCauseMatches(cause, el, shift))) return true;
// Scroll operations may change layout inside their actual scroll box.
// Ignore document-wide ancestors so an unrelated page shift is not claimed.
for (const region of op.scrollAncestors || []) {
@@ -195,14 +190,32 @@
if (layoutAgentOperation(op, el, shift)) return op.userAt;
}
for (const op of operations.values()) if (layoutAgentOperation(op, el, shift)) return op.userAt;
+ pruneAgentLayoutHistory();
+ const cause = agentLayoutHistory.find(entry => lastUserAt <= entry.userAt && layoutCauseMatches(entry, el, shift));
+ if (cause) return cause.userAt;
return undefined;
}
+ function pruneAgentLayoutHistory() {
+ const now = Date.now();
+ for (let i = agentLayoutHistory.length - 1; i >= 0; i--) if (agentLayoutHistory[i].until < now) agentLayoutHistory.splice(i, 1);
+ }
+ function layoutCauseMatches(cause, el, shift) {
+ if (Date.now() > cause.until || cause.parent !== el.parentElement || !shift?.previousRect || !shift?.currentRect) return false;
+ const sourceTopDelta = shift.currentRect.y - shift.previousRect.y;
+ return Math.abs(sourceTopDelta) > 2 && Math.sign(sourceTopDelta) === Math.sign(cause.heightDelta)
+ && Math.abs(sourceTopDelta) <= Math.abs(cause.heightDelta) + 8;
+ }
function rememberAgentLayout(op) {
if (!op?.target) return;
+ pruneAgentLayoutHistory();
const before = op.layoutRect || op.targetRectAtDispatch, after = rectFor(op.target);
if (before && after && Math.abs(after.height - before.height) > 2) {
const causes = op.layoutCauses ||= [];
- causes.push({ parent: op.target.parentElement, heightDelta: after.height - before.height, until: Date.now() + 3000 });
+ const cause = { parent: op.target.parentElement, heightDelta: after.height - before.height,
+ userAt: op.userAt, until: Date.now() + 3000 };
+ causes.push(cause);
+ agentLayoutHistory.push(cause);
+ if (agentLayoutHistory.length > 256) agentLayoutHistory.splice(0, agentLayoutHistory.length - 256);
if (causes.length > 8) causes.splice(0, causes.length - 8);
}
if (after) op.layoutRect = after;
@@ -348,6 +361,12 @@
trackControl(el);
controlSignatures.set(el, controlState(el));
}
+ if (el.tagName === 'SELECT') {
+ for (const option of liveControls) if (option.tagName === 'OPTION' && option.closest('select') === el) {
+ controlSignatures.set(option, controlState(option));
+ signatures.set(option, signature(option));
+ }
+ }
signatures.set(el, signature(el));
}
function signature(el) {
@@ -495,6 +514,21 @@
return false;
}));
}
+ function hasVisibleAriaLabelledbyConsumer(el) {
+ const ids = new Set();
+ for (let node = el; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (node.id) ids.add(node.id);
+ }
+ if (!ids.size) return false;
+ let consumers;
+ try { consumers = document.querySelectorAll('[aria-labelledby]'); } catch { return false; }
+ for (const consumer of consumers) {
+ if (ignored(consumer) || !visible(consumer)) continue;
+ const references = (consumer.getAttribute('aria-labelledby') || '').trim().split(/\s+/);
+ if (references.some(id => ids.has(id))) return true;
+ }
+ return false;
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -539,12 +573,13 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
- const editableCharacterData = record.type === 'characterData' && editable(el);
+ const editableTextMutation = editable(el) && (record.type === 'characterData'
+ || (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
if (ignored(el)) continue;
// Native input already reports user edits, and agent input's synchronous
// DOM writes are attributed to its dispatch. Keep those paths deduplicated
// while observing independent page-script changes to editable text.
- if (editableCharacterData && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
+ if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& el.getAnimations?.().some(animation => animation.playState === 'running')) continue;
if (++measured > 300) {
@@ -563,12 +598,15 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
+ const hiddenAccessibleNameChanged = !visible(el)
+ && ['characterData', 'childList'].includes(record.type)
+ && hasVisibleAriaLabelledbyConsumer(el);
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
- || editableCharacterData
+ || editableTextMutation || hiddenAccessibleNameChanged
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
if (record.type === 'childList') {
- identityChanged = subtreeChanged(record, el);
+ identityChanged ||= subtreeChanged(record, el);
for (const node of record.addedNodes) {
if (node.nodeType !== 1 || ignored(node)) continue;
if (seedBudget.remaining > 0) { seedBudget.remaining--; signatures.set(node, signature(node)); }
@@ -641,7 +679,8 @@
try {
const guard = JSON.parse(String(event.detail));
const op = operations.get(guard.operationId);
- const phase = ['focus', 'click'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const phase = ['focus', 'click', 'input'].includes(guard.dispatchPhase) ? guard.dispatchPhase : 'dom';
+ const phaseKind = phase === 'input' ? 'input' : phase === 'dom' ? 'dom' : 'click';
const actionPhase = phase !== 'dom';
const phaseAlreadyConsumed = phase === 'dom'
? op?.domDispatchConsumed === true
@@ -650,17 +689,19 @@
flushPendingMutations();
sampleFormControls(op?.target);
if (!active || guard.runToken !== runToken || guard.documentToken !== documentToken || guard.revision !== revision
- || !op || phaseAlreadyConsumed || (actionPhase ? op.kind !== 'click' : op.kind !== 'dom')
+ || !op || phaseAlreadyConsumed || op.kind !== phaseKind
+ || (phase === 'input' && elementFor(event) !== op.target && elementFor(event) !== op.focusTarget)
|| op.preparedRevision !== revision || domTimer || unreported || lastUserAt > op.userAt) {
event.preventDefault(); return;
}
if (actionPhase) {
op.domDispatchPhases ||= new Set();
op.domDispatchPhases.add(phase);
- if (!op.dispatched) dispatch({ operationId: op.operationId, kind: 'click', runToken });
+ if (!op.dispatched) dispatch({ operationId: op.operationId, kind: phaseKind, runToken });
+ if (phase === 'input') op.synchronous = true;
} else op.domDispatchConsumed = true;
const marker = { userAt: lastUserAt }; agentTurn = marker;
- setTimeout(() => { if (agentTurn === marker) agentTurn = null; }, 0);
+ setTimeout(() => { if (phase === 'input') op.synchronous = false; if (agentTurn === marker) agentTurn = null; }, 0);
} catch { event.stopImmediatePropagation(); event.preventDefault(); /* Only a current prepared operation can mark a DOM write. */ }
}, false);
listen(document, 'webbrain-agent-scroll-dispatch', event => {
@@ -815,6 +856,7 @@
clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
+ agentLayoutHistory.length = 0;
roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
}
async function requestState() {
@@ -911,7 +953,12 @@
const previous = op.synchronous;
op.synchronous = true;
try { return callback(); }
- finally { op.synchronous = previous; }
+ finally {
+ op.synchronous = previous;
+ // Native editing can move following siblings; record the causal size
+ // change now, before the browser reports its layout shift asynchronously.
+ rememberAgentLayout(op);
+ }
}
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
'hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 1e24b1f5a..e6a354d38 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -896,21 +896,27 @@ for (const path of ['local', 'memory']) {
const originalMessage = api.tabs.sendMessage;
try {
const binding = bind(agent, tab), child = bind(agent, tab, 2, 'upload-doc', 'upload-token');
- const client = new CDPClient();
+ const client = new CDPClient(), monitorGuards = [], runtimeCalls = [];
client._pageAgentObjectTarget = async () => {
if (intervention === 'preparation') binding.send({ kind: 'click' });
return { documentToken: child.state.documentToken, documentRevision: 0, nativeTarget: 'upload-target' };
};
api.tabs.sendMessage = async (_tab, message, options) => {
+ if (message.action === 'page_monitor_prepare') return { ready: true };
if (message.action === 'page_monitor_dispatch') {
assert.equal(options.frameId, 2);
notices.push(message);
if (intervention === 'handshake') binding.send({ kind: 'click' });
+ const guard = { runToken: message.params.runToken, documentToken: child.state.documentToken,
+ revision: 0, operationId: message.params.operationId };
+ monitorGuards.push(guard);
+ return { ready: true, guard };
}
return {};
};
client.sendCommand = async (_tab, method, params) => {
if (method === 'DOM.setFileInputFiles' || method === 'Runtime.callFunctionOn') mutations.push(method);
+ if (method === 'Runtime.callFunctionOn') runtimeCalls.push(params);
return { result: { value: { success: true, dispatched: true } } };
};
let started = false;
@@ -922,6 +928,12 @@ for (const path of ['local', 'memory']) {
assert.equal(started, true);
assert.equal(mutations.length, 1);
assert.equal(notices.length, 1);
+ if (path === 'memory') {
+ const call = runtimeCalls[0];
+ assert.deepEqual(call.arguments.at(-1), { value: monitorGuards[0] });
+ assert.ok(call.functionDeclaration.indexOf("beforePageAgentDomAction(pageGuard, 'input', this)")
+ < call.functionDeclaration.indexOf('this.files = transfer.files'));
+ }
} else {
await assert.rejects(upload, { code: 'page_feedback_pending' });
assert.equal(started, false);
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 2065b709a..33305ab46 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1095,6 +1095,19 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: native select input does not emit a delayed duplicate option change`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => { feedback = []; });
+ await page.locator('#select').selectOption('B');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'input'), null, { timeout: 1000 });
+ await page.waitForTimeout(350);
+ const observed = await page.evaluate(() => feedback);
+ assert.equal(observed.some(event => event.kind === 'dom' && event.target === 'option'), false,
+ 'Sampling the option property after its user input must not replan the run a second time');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: native markers reject replay, tampering and copied targets`, async () => {
const { browser, page } = await fixture(engine, build);
try {
@@ -1389,7 +1402,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
if (behavior === 'editor-handles') { event.preventDefault(); editor.append(' handled'); }
};
}, behavior);
- await page.waitForTimeout(200);
+ await page.waitForTimeout(400);
await page.evaluate(() => { feedback = []; });
const result = await page.evaluate(params => typeRichText(params), {
text: behavior === 'replace' ? 'New\ntext' : ' added', clear: behavior === 'replace',
@@ -1405,8 +1418,8 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
if (behavior === 'append') assert.equal(await page.evaluate(() => document.getElementById('rich-editor').contains(originalStrong)), true);
else assert.equal(result.value, 'New\ntext');
}
- await page.waitForTimeout(200);
- assert.deepEqual(await page.evaluate(() => feedback), [], 'Own beforeinput, input and DOM effects must not trigger replanning');
+ await page.waitForTimeout(400);
+ assert.deepEqual(await page.evaluate(() => feedback), [], `Own beforeinput, input and DOM effects must not trigger replanning (${behavior})`);
}
await page.evaluate(() => {
const editor = document.getElementById('rich-editor'); editor.onbeforeinput = null; editor.textContent = 'Ready';
@@ -1591,6 +1604,112 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
assert.ok(observations.some(event => event.source === 'page' && event.target.startsWith('div#programmatic-editor')));
assert.equal(JSON.stringify(observations).includes('Page updated this editor'), false,
'Editable content must invalidate state without being included in feedback');
+
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'editable-replacement', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'editable-replacement', kind: 'click', selector: '#agent', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ await page.evaluate(() => { document.getElementById('programmatic-editor').textContent = 'Page replaced the editor value'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ const replacementObservations = await page.evaluate(() => feedback);
+ assert.ok(replacementObservations.some(event => event.source === 'page'
+ && event.target.startsWith('div#programmatic-editor')),
+ 'Replacing an editable text node must advance the page revision');
+ assert.equal(JSON.stringify(replacementObservations).includes('Page replaced the editor value'), false,
+ 'Replaced editable text must remain private');
+ const ready = await page.evaluate(async () => {
+ deliver('page_monitor_dispatch', { operationId: 'editable-replacement', kind: 'click', selector: '#agent' });
+ await new Promise(resolve => setTimeout(resolve, 0));
+ return lastMonitorResponse;
+ });
+ assert.equal(ready.pageFeedbackPending, true, 'A prepared action must not survive editable text replacement');
+ assert.ok(replacementObservations.some(observation => observation.revision > guard.revision));
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: hidden aria-labelledby text changes invalidate visible targets`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const label = document.createElement('span'); label.id = 'hidden-accessible-label'; label.textContent = 'Initial label';
+ label.style.display = 'none';
+ const button = document.createElement('button'); button.id = 'labelled-button'; button.setAttribute('aria-labelledby', label.id);
+ document.body.append(label, button);
+ });
+ await page.waitForTimeout(180);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'hidden-label', tool: 'click', selector: '#labelled-button' });
+ deliver('page_monitor_dispatch', { operationId: 'hidden-label', kind: 'click', selector: '#labelled-button', fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ await page.evaluate(() => { document.getElementById('hidden-accessible-label').textContent = 'Updated label'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ const observations = await page.evaluate(() => feedback);
+ assert.ok(observations.some(event => event.source === 'page'));
+ assert.equal(JSON.stringify(observations).includes('Updated label'), false, 'Hidden accessible-name text must not be sent');
+ const accepted = await page.evaluate(value => {
+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), bubbles: true, composed: true, cancelable: true,
+ });
+ return document.getElementById('labelled-button').dispatchEvent(event);
+ }, guard);
+ assert.equal(accepted, false, 'Changing a hidden accessible label must invalidate the prepared click');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: guarded synthetic file input events are attributed once`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const input = document.createElement('input'); input.id = 'synthetic-upload'; input.type = 'file'; document.body.append(input);
+ });
+ await page.waitForTimeout(200);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'synthetic-upload', tool: 'upload_file', selector: '#synthetic-upload' });
+ deliver('page_monitor_dispatch', { operationId: 'synthetic-upload', kind: 'input', selector: '#synthetic-upload',
+ eventTypes: ['input', 'change'], fenceOnly: true });
+ return lastMonitorResponse.guard;
+ });
+ const accepted = await page.locator('#synthetic-upload').evaluate((input, value) => {
+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'input' }), bubbles: true, composed: true, cancelable: true,
+ });
+ if (!input.dispatchEvent(event)) return false;
+ const transfer = new DataTransfer(); transfer.items.add(new File(['fixture'], 'fixture.txt', { type: 'text/plain' }));
+ input.files = transfer.files;
+ input.dispatchEvent(new Event('input', { bubbles: true }));
+ input.dispatchEvent(new Event('change', { bubbles: true }));
+ return true;
+ }, guard);
+ assert.equal(accepted, true);
+ await page.waitForTimeout(200);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'The guarded synthetic upload must not interrupt its own run');
+
+ const staleGuard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'stale-synthetic-upload', tool: 'upload_file', selector: '#synthetic-upload' });
+ deliver('page_monitor_dispatch', { operationId: 'stale-synthetic-upload', kind: 'input', selector: '#synthetic-upload',
+ eventTypes: ['input', 'change'], fenceOnly: true });
+ const value = lastMonitorResponse.guard;
+ document.getElementById('status').textContent = 'Page changed before upload dispatch';
+ return value;
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ const staleAccepted = await page.locator('#synthetic-upload').evaluate((input, value) => {
+ let inputEvents = 0;
+ input.addEventListener('input', () => inputEvents++, { once: true });
+ const event = new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'input' }), bubbles: true, composed: true, cancelable: true,
+ });
+ const okay = input.dispatchEvent(event);
+ if (okay) input.dispatchEvent(new Event('input', { bubbles: true }));
+ return { okay, inputEvents };
+ }, staleGuard);
+ assert.deepEqual(staleAccepted, { okay: false, inputEvents: 0 }, 'Stale upload guards must stop the synthetic input sequence');
} finally { await browser.close(); }
});