Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix keyboard activation feedback attribution

Barack Sokullu6d312d8b4ec0
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 22a6da03f..fd2868ba7 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -283,8 +283,10 @@
const match = op => {
if (event && !event.isTrusted && !op.synchronous) return null;
if (event) {
- if (op.eventTypes && !op.eventTypes.has(event.type)
- && !(kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el))) return null;
+ const keyboardClick = kind === 'click' && event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget
+ && related(op.keyboardClickTarget, el);
+ const submitClick = kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el);
+ if (op.eventTypes && !op.eventTypes.has(event.type) && !keyboardClick && !submitClick) return null;
// One native dispatch owns one occurrence of each input phase. A later
// human action on the same target must not fit that expectation again.
if (!op.synchronous && event.type !== 'pointermove' && op.seenEvents?.has(event.type)) return null;
@@ -300,13 +302,18 @@
}
if (!op.dispatched) continue;
if (lastUserAt > op.userAt) continue;
+ // Keyboard-activated clicks can report detail=1 in Firefox. Do not let
+ // the temporary click expectation absorb a physical pointer sequence.
+ if (kind === 'click' && op.keyboardClickTarget && event && event.type !== 'click') continue;
// Moving agent focus commits the previous field's native change event.
if (kind === 'input' && event?.type === 'change' && op.blurTarget === el) { const found = match(op); if (found) return found; }
if (kind === 'scroll' && op.blurTarget === el && editable(el)) { const found = match(op); if (found) return found; }
if (!op.kinds.has(kind)) continue;
if (['wheel', 'touchmove'].includes(event?.type) && !op.nativeWheel) continue;
const target = op.target || (op.focused ? document.activeElement : null);
- if (kind === 'click' && event?.detail === 0 && op.submitTarget && related(op.submitTarget, el)) {
+ if (kind === 'click' && event?.type === 'click' && event?.isTrusted === true
+ && ((event.detail === 0 && op.submitTarget && related(op.submitTarget, el))
+ || (op.keyboardClickTarget && related(op.keyboardClickTarget, el)))) {
const found = match(op); if (found) return found;
}
if (kind === 'scroll' && op.scrollAncestors?.has(el)) { const found = match(op); if (found) return found; }
@@ -580,6 +587,20 @@
|| (root === document ? document.getElementById(previousFor) : null);
return visibleControl(priorControl);
}
+ const descendantTextNameRoles = new Set([
+ 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'tab', 'treeitem', 'row', 'gridcell', 'cell', 'listitem',
+ ]);
+ const nonImplicitSubmitInputTypes = new Set([
+ 'button', 'submit', 'reset', 'checkbox', 'radio', 'file', 'image', 'hidden', 'color', 'range',
+ ]);
+ function hasVisibleTextNameConsumer(el) {
+ for (let node = el.parentElement || el.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (!visible(node)) continue;
+ const tag = node.tagName, role = (node.getAttribute('role') || '').toLowerCase();
+ if (['BUTTON', 'A', 'SUMMARY', 'LI'].includes(tag) || /^H[1-6]$/.test(tag) || descendantTextNameRoles.has(role)) return true;
+ }
+ return false;
+ }
function stylesheetNode(node) {
if (!(node instanceof Element)) return false;
return node.tagName === 'STYLE' || (node.tagName === 'LINK'
@@ -725,6 +746,7 @@
const hiddenAccessibleNameChanged = !visible(el)
&& ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))
|| (hiddenAriaLabelIdMutation && hasVisibleAriaLabelledbyConsumer(el, record.oldValue))
+ || (accessibleNameContentMutation && hasVisibleTextNameConsumer(el))
|| ((accessibleNameContentMutation || nativeLabelAssociationMutation)
&& hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));
let identityChanged = (record.type === 'popover' && record.stateChanged)
@@ -855,8 +877,21 @@
const marker = { userAt: op.userAt };
agentTurn = marker;
const enterFormSubmit = event.type === 'keydown' && event.key === 'Enter' && op.kind === 'input'
- && el?.tagName !== 'TEXTAREA' && !el?.isContentEditable && !!el?.form;
- if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit) {
+ && !el?.isContentEditable && !!el?.form
+ && ((el.tagName === 'INPUT' && !nonImplicitSubmitInputTypes.has(el.type))
+ || el.matches?.('button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]'));
+ if (event.type === 'keydown' && !enterFormSubmit) {
+ const key = event.key === 'Spacebar' ? ' ' : event.key;
+ const keyboardClickTarget = event.composedPath().find(node => node instanceof Element && (
+ (key === 'Enter' && node.matches('a[href],area[href]'))
+ || (['Enter', ' '].includes(key) && node.matches('button,summary,input[type="button"],input[type="submit"],input[type="reset"],input[type="image"]'))
+ || (key === ' ' && node.matches('input[type="checkbox"],input[type="radio"]'))
+ ));
+ if (keyboardClickTarget) { op.keyboardClickTarget = keyboardClickTarget; op.kinds.add('click'); }
+ }
+ const keyboardClick = event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget
+ && related(op.keyboardClickTarget, el);
+ if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit || keyboardClick) {
const path = event.composedPath();
const link = enterFormSubmit ? null : path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
let submitter = path.find(node => node instanceof Element
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 22a6da03f..fd2868ba7 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -283,8 +283,10 @@
const match = op => {
if (event && !event.isTrusted && !op.synchronous) return null;
if (event) {
- if (op.eventTypes && !op.eventTypes.has(event.type)
- && !(kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el))) return null;
+ const keyboardClick = kind === 'click' && event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget
+ && related(op.keyboardClickTarget, el);
+ const submitClick = kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el);
+ if (op.eventTypes && !op.eventTypes.has(event.type) && !keyboardClick && !submitClick) return null;
// One native dispatch owns one occurrence of each input phase. A later
// human action on the same target must not fit that expectation again.
if (!op.synchronous && event.type !== 'pointermove' && op.seenEvents?.has(event.type)) return null;
@@ -300,13 +302,18 @@
}
if (!op.dispatched) continue;
if (lastUserAt > op.userAt) continue;
+ // Keyboard-activated clicks can report detail=1 in Firefox. Do not let
+ // the temporary click expectation absorb a physical pointer sequence.
+ if (kind === 'click' && op.keyboardClickTarget && event && event.type !== 'click') continue;
// Moving agent focus commits the previous field's native change event.
if (kind === 'input' && event?.type === 'change' && op.blurTarget === el) { const found = match(op); if (found) return found; }
if (kind === 'scroll' && op.blurTarget === el && editable(el)) { const found = match(op); if (found) return found; }
if (!op.kinds.has(kind)) continue;
if (['wheel', 'touchmove'].includes(event?.type) && !op.nativeWheel) continue;
const target = op.target || (op.focused ? document.activeElement : null);
- if (kind === 'click' && event?.detail === 0 && op.submitTarget && related(op.submitTarget, el)) {
+ if (kind === 'click' && event?.type === 'click' && event?.isTrusted === true
+ && ((event.detail === 0 && op.submitTarget && related(op.submitTarget, el))
+ || (op.keyboardClickTarget && related(op.keyboardClickTarget, el)))) {
const found = match(op); if (found) return found;
}
if (kind === 'scroll' && op.scrollAncestors?.has(el)) { const found = match(op); if (found) return found; }
@@ -580,6 +587,20 @@
|| (root === document ? document.getElementById(previousFor) : null);
return visibleControl(priorControl);
}
+ const descendantTextNameRoles = new Set([
+ 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'tab', 'treeitem', 'row', 'gridcell', 'cell', 'listitem',
+ ]);
+ const nonImplicitSubmitInputTypes = new Set([
+ 'button', 'submit', 'reset', 'checkbox', 'radio', 'file', 'image', 'hidden', 'color', 'range',
+ ]);
+ function hasVisibleTextNameConsumer(el) {
+ for (let node = el.parentElement || el.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) {
+ if (!visible(node)) continue;
+ const tag = node.tagName, role = (node.getAttribute('role') || '').toLowerCase();
+ if (['BUTTON', 'A', 'SUMMARY', 'LI'].includes(tag) || /^H[1-6]$/.test(tag) || descendantTextNameRoles.has(role)) return true;
+ }
+ return false;
+ }
function stylesheetNode(node) {
if (!(node instanceof Element)) return false;
return node.tagName === 'STYLE' || (node.tagName === 'LINK'
@@ -725,6 +746,7 @@
const hiddenAccessibleNameChanged = !visible(el)
&& ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))
|| (hiddenAriaLabelIdMutation && hasVisibleAriaLabelledbyConsumer(el, record.oldValue))
+ || (accessibleNameContentMutation && hasVisibleTextNameConsumer(el))
|| ((accessibleNameContentMutation || nativeLabelAssociationMutation)
&& hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));
let identityChanged = (record.type === 'popover' && record.stateChanged)
@@ -855,8 +877,21 @@
const marker = { userAt: op.userAt };
agentTurn = marker;
const enterFormSubmit = event.type === 'keydown' && event.key === 'Enter' && op.kind === 'input'
- && el?.tagName !== 'TEXTAREA' && !el?.isContentEditable && !!el?.form;
- if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit) {
+ && !el?.isContentEditable && !!el?.form
+ && ((el.tagName === 'INPUT' && !nonImplicitSubmitInputTypes.has(el.type))
+ || el.matches?.('button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]'));
+ if (event.type === 'keydown' && !enterFormSubmit) {
+ const key = event.key === 'Spacebar' ? ' ' : event.key;
+ const keyboardClickTarget = event.composedPath().find(node => node instanceof Element && (
+ (key === 'Enter' && node.matches('a[href],area[href]'))
+ || (['Enter', ' '].includes(key) && node.matches('button,summary,input[type="button"],input[type="submit"],input[type="reset"],input[type="image"]'))
+ || (key === ' ' && node.matches('input[type="checkbox"],input[type="radio"]'))
+ ));
+ if (keyboardClickTarget) { op.keyboardClickTarget = keyboardClickTarget; op.kinds.add('click'); }
+ }
+ const keyboardClick = event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget
+ && related(op.keyboardClickTarget, el);
+ if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit || keyboardClick) {
const path = event.composedPath();
const link = enterFormSubmit ? null : path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
let submitter = path.find(node => node instanceof Element
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index b505a74d1..48ad98b7b 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -2029,6 +2029,76 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: boxless text changes invalidate visible descendant-name controls`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const button = document.createElement('button'); button.id = 'boxless-name-button';
+ const label = document.createElement('span'); label.style.display = 'contents'; label.textContent = 'Initial action';
+ button.append(label); document.body.append(button);
+ });
+ await page.waitForTimeout(300);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'boxless-name', tool: 'click', selector: '#boxless-name-button' });
+ deliver('page_monitor_dispatch', { operationId: 'boxless-name', kind: 'click', selector: '#boxless-name-button' });
+ return lastMonitorResponse.guard;
+ });
+ await page.locator('#boxless-name-button span').evaluate(label => { label.textContent = 'Updated action'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated action'), false,
+ 'The changed accessible name must not be copied into feedback');
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, 'Changing text in a boxless name descendant must invalidate the prepared action');
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: Enter and Space keyboard activations stay attributed to the agent`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const link = document.createElement('a'); link.id = 'keyboard-link'; link.href = '/keyboard-destination';
+ link.textContent = 'Open page'; link.addEventListener('click', event => event.preventDefault());
+ const button = document.createElement('button'); button.id = 'keyboard-button'; button.type = 'button';
+ button.textContent = 'Run action'; button.addEventListener('click', event => event.preventDefault());
+ document.body.append(link, button);
+ });
+ await page.waitForTimeout(300);
+ const linkDispatch = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'keyboard-link', tool: 'press_keys', selector: '#keyboard-link' });
+ deliver('page_monitor_dispatch', { operationId: 'keyboard-link', kind: 'input', selector: '#keyboard-link' });
+ return lastMonitorResponse;
+ });
+ assert.equal(linkDispatch.ready, true, JSON.stringify(linkDispatch));
+ await page.locator('#keyboard-link').press('Enter');
+ await page.waitForFunction(() => feedback.some(event => event.source === 'agent' && event.operation === 'click'), null, { timeout: 1000 });
+ const linkEvents = await page.evaluate(() => feedback);
+ assert.equal(linkEvents.some(event => event.source === 'agent' && event.operation === 'click'
+ && event.navigationUrl === 'https://monitor.test/keyboard-destination'), true, JSON.stringify(linkEvents));
+ assert.equal(linkEvents.some(event => event.kind === 'click' && event.source === 'user'), false, JSON.stringify(linkEvents));
+
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'keyboard-button', tool: 'press_keys', selector: '#keyboard-button' });
+ deliver('page_monitor_dispatch', { operationId: 'keyboard-button', kind: 'input', selector: '#keyboard-button' });
+ });
+ await page.locator('#keyboard-button').press('Space');
+ await page.waitForFunction(() => feedback.some(event => event.source === 'agent' && event.operation === 'click'), null, { timeout: 1000 });
+ const buttonEvents = await page.evaluate(() => feedback);
+ assert.equal(buttonEvents.some(event => event.source === 'agent' && event.operation === 'click'), true, JSON.stringify(buttonEvents));
+ assert.equal(buttonEvents.some(event => event.kind === 'click' && event.source === 'user'), false, JSON.stringify(buttonEvents));
+
+ await page.evaluate(() => { feedback = []; });
+ await page.locator('#keyboard-button').click();
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'click' && event.source === 'user'), null, { timeout: 1000 });
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'click' && event.source === 'user'), true,
+ 'A later pointer click must remain user activity after keyboard attribution');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: aria-modal changes invalidate prepared page context`, async () => {
const { browser, page } = await fixture(engine, build);
try {