Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix keyboard activation feedback attribution
Barack Sokullu6d312d8b4ec0
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 22a6da03f..fd2868ba7 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -283,8 +283,10 @@const match = op => {if (event && !event.isTrusted && !op.synchronous) return null;if (event) {- if (op.eventTypes && !op.eventTypes.has(event.type)- && !(kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el))) return null;+ const keyboardClick = kind === 'click' && event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget+ && related(op.keyboardClickTarget, el);+ const submitClick = kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el);+ if (op.eventTypes && !op.eventTypes.has(event.type) && !keyboardClick && !submitClick) return null;// One native dispatch owns one occurrence of each input phase. A later// human action on the same target must not fit that expectation again.if (!op.synchronous && event.type !== 'pointermove' && op.seenEvents?.has(event.type)) return null;@@ -300,13 +302,18 @@}if (!op.dispatched) continue;if (lastUserAt > op.userAt) continue;+ // Keyboard-activated clicks can report detail=1 in Firefox. Do not let+ // the temporary click expectation absorb a physical pointer sequence.+ if (kind === 'click' && op.keyboardClickTarget && event && event.type !== 'click') continue;// Moving agent focus commits the previous field's native change event.if (kind === 'input' && event?.type === 'change' && op.blurTarget === el) { const found = match(op); if (found) return found; }if (kind === 'scroll' && op.blurTarget === el && editable(el)) { const found = match(op); if (found) return found; }if (!op.kinds.has(kind)) continue;if (['wheel', 'touchmove'].includes(event?.type) && !op.nativeWheel) continue;const target = op.target || (op.focused ? document.activeElement : null);- if (kind === 'click' && event?.detail === 0 && op.submitTarget && related(op.submitTarget, el)) {+ if (kind === 'click' && event?.type === 'click' && event?.isTrusted === true+ && ((event.detail === 0 && op.submitTarget && related(op.submitTarget, el))+ || (op.keyboardClickTarget && related(op.keyboardClickTarget, el)))) {const found = match(op); if (found) return found;}if (kind === 'scroll' && op.scrollAncestors?.has(el)) { const found = match(op); if (found) return found; }@@ -580,6 +587,20 @@|| (root === document ? document.getElementById(previousFor) : null);return visibleControl(priorControl);}+ const descendantTextNameRoles = new Set([+ 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'tab', 'treeitem', 'row', 'gridcell', 'cell', 'listitem',+ ]);+ const nonImplicitSubmitInputTypes = new Set([+ 'button', 'submit', 'reset', 'checkbox', 'radio', 'file', 'image', 'hidden', 'color', 'range',+ ]);+ function hasVisibleTextNameConsumer(el) {+ for (let node = el.parentElement || el.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) {+ if (!visible(node)) continue;+ const tag = node.tagName, role = (node.getAttribute('role') || '').toLowerCase();+ if (['BUTTON', 'A', 'SUMMARY', 'LI'].includes(tag) || /^H[1-6]$/.test(tag) || descendantTextNameRoles.has(role)) return true;+ }+ return false;+ }function stylesheetNode(node) {if (!(node instanceof Element)) return false;return node.tagName === 'STYLE' || (node.tagName === 'LINK'@@ -725,6 +746,7 @@const hiddenAccessibleNameChanged = !visible(el)&& ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))|| (hiddenAriaLabelIdMutation && hasVisibleAriaLabelledbyConsumer(el, record.oldValue))+ || (accessibleNameContentMutation && hasVisibleTextNameConsumer(el))|| ((accessibleNameContentMutation || nativeLabelAssociationMutation)&& hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));let identityChanged = (record.type === 'popover' && record.stateChanged)@@ -855,8 +877,21 @@const marker = { userAt: op.userAt };agentTurn = marker;const enterFormSubmit = event.type === 'keydown' && event.key === 'Enter' && op.kind === 'input'- && el?.tagName !== 'TEXTAREA' && !el?.isContentEditable && !!el?.form;- if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit) {+ && !el?.isContentEditable && !!el?.form+ && ((el.tagName === 'INPUT' && !nonImplicitSubmitInputTypes.has(el.type))+ || el.matches?.('button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]'));+ if (event.type === 'keydown' && !enterFormSubmit) {+ const key = event.key === 'Spacebar' ? ' ' : event.key;+ const keyboardClickTarget = event.composedPath().find(node => node instanceof Element && (+ (key === 'Enter' && node.matches('a[href],area[href]'))+ || (['Enter', ' '].includes(key) && node.matches('button,summary,input[type="button"],input[type="submit"],input[type="reset"],input[type="image"]'))+ || (key === ' ' && node.matches('input[type="checkbox"],input[type="radio"]'))+ ));+ if (keyboardClickTarget) { op.keyboardClickTarget = keyboardClickTarget; op.kinds.add('click'); }+ }+ const keyboardClick = event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget+ && related(op.keyboardClickTarget, el);+ if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit || keyboardClick) {const path = event.composedPath();const link = enterFormSubmit ? null : path.find(node => node instanceof Element && node.matches('a[href],area[href]'));let submitter = path.find(node => node instanceof Elementdiff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 22a6da03f..fd2868ba7 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -283,8 +283,10 @@const match = op => {if (event && !event.isTrusted && !op.synchronous) return null;if (event) {- if (op.eventTypes && !op.eventTypes.has(event.type)- && !(kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el))) return null;+ const keyboardClick = kind === 'click' && event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget+ && related(op.keyboardClickTarget, el);+ const submitClick = kind === 'click' && event.detail === 0 && op.submitTarget && related(op.submitTarget, el);+ if (op.eventTypes && !op.eventTypes.has(event.type) && !keyboardClick && !submitClick) return null;// One native dispatch owns one occurrence of each input phase. A later// human action on the same target must not fit that expectation again.if (!op.synchronous && event.type !== 'pointermove' && op.seenEvents?.has(event.type)) return null;@@ -300,13 +302,18 @@}if (!op.dispatched) continue;if (lastUserAt > op.userAt) continue;+ // Keyboard-activated clicks can report detail=1 in Firefox. Do not let+ // the temporary click expectation absorb a physical pointer sequence.+ if (kind === 'click' && op.keyboardClickTarget && event && event.type !== 'click') continue;// Moving agent focus commits the previous field's native change event.if (kind === 'input' && event?.type === 'change' && op.blurTarget === el) { const found = match(op); if (found) return found; }if (kind === 'scroll' && op.blurTarget === el && editable(el)) { const found = match(op); if (found) return found; }if (!op.kinds.has(kind)) continue;if (['wheel', 'touchmove'].includes(event?.type) && !op.nativeWheel) continue;const target = op.target || (op.focused ? document.activeElement : null);- if (kind === 'click' && event?.detail === 0 && op.submitTarget && related(op.submitTarget, el)) {+ if (kind === 'click' && event?.type === 'click' && event?.isTrusted === true+ && ((event.detail === 0 && op.submitTarget && related(op.submitTarget, el))+ || (op.keyboardClickTarget && related(op.keyboardClickTarget, el)))) {const found = match(op); if (found) return found;}if (kind === 'scroll' && op.scrollAncestors?.has(el)) { const found = match(op); if (found) return found; }@@ -580,6 +587,20 @@|| (root === document ? document.getElementById(previousFor) : null);return visibleControl(priorControl);}+ const descendantTextNameRoles = new Set([+ 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'tab', 'treeitem', 'row', 'gridcell', 'cell', 'listitem',+ ]);+ const nonImplicitSubmitInputTypes = new Set([+ 'button', 'submit', 'reset', 'checkbox', 'radio', 'file', 'image', 'hidden', 'color', 'range',+ ]);+ function hasVisibleTextNameConsumer(el) {+ for (let node = el.parentElement || el.getRootNode?.().host; node; node = node.parentElement || node.getRootNode?.().host) {+ if (!visible(node)) continue;+ const tag = node.tagName, role = (node.getAttribute('role') || '').toLowerCase();+ if (['BUTTON', 'A', 'SUMMARY', 'LI'].includes(tag) || /^H[1-6]$/.test(tag) || descendantTextNameRoles.has(role)) return true;+ }+ return false;+ }function stylesheetNode(node) {if (!(node instanceof Element)) return false;return node.tagName === 'STYLE' || (node.tagName === 'LINK'@@ -725,6 +746,7 @@const hiddenAccessibleNameChanged = !visible(el)&& ((accessibleNameContentMutation && hasVisibleAriaLabelledbyConsumer(el))|| (hiddenAriaLabelIdMutation && hasVisibleAriaLabelledbyConsumer(el, record.oldValue))+ || (accessibleNameContentMutation && hasVisibleTextNameConsumer(el))|| ((accessibleNameContentMutation || nativeLabelAssociationMutation)&& hasVisibleNativeLabelConsumer(el, nativeLabelAssociationMutation ? record.oldValue : '')));let identityChanged = (record.type === 'popover' && record.stateChanged)@@ -855,8 +877,21 @@const marker = { userAt: op.userAt };agentTurn = marker;const enterFormSubmit = event.type === 'keydown' && event.key === 'Enter' && op.kind === 'input'- && el?.tagName !== 'TEXTAREA' && !el?.isContentEditable && !!el?.form;- if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit) {+ && !el?.isContentEditable && !!el?.form+ && ((el.tagName === 'INPUT' && !nonImplicitSubmitInputTypes.has(el.type))+ || el.matches?.('button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]'));+ if (event.type === 'keydown' && !enterFormSubmit) {+ const key = event.key === 'Spacebar' ? ' ' : event.key;+ const keyboardClickTarget = event.composedPath().find(node => node instanceof Element && (+ (key === 'Enter' && node.matches('a[href],area[href]'))+ || (['Enter', ' '].includes(key) && node.matches('button,summary,input[type="button"],input[type="submit"],input[type="reset"],input[type="image"]'))+ || (key === ' ' && node.matches('input[type="checkbox"],input[type="radio"]'))+ ));+ if (keyboardClickTarget) { op.keyboardClickTarget = keyboardClickTarget; op.kinds.add('click'); }+ }+ const keyboardClick = event.type === 'click' && event.isTrusted === true && op.keyboardClickTarget+ && related(op.keyboardClickTarget, el);+ if ((op.navigationCandidate && ['click', 'pointerdown'].includes(event.type)) || enterFormSubmit || keyboardClick) {const path = event.composedPath();const link = enterFormSubmit ? null : path.find(node => node instanceof Element && node.matches('a[href],area[href]'));let submitter = path.find(node => node instanceof Elementdiff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex b505a74d1..48ad98b7b 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -2029,6 +2029,76 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: boxless text changes invalidate visible descendant-name controls`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const button = document.createElement('button'); button.id = 'boxless-name-button';+ const label = document.createElement('span'); label.style.display = 'contents'; label.textContent = 'Initial action';+ button.append(label); document.body.append(button);+ });+ await page.waitForTimeout(300);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'boxless-name', tool: 'click', selector: '#boxless-name-button' });+ deliver('page_monitor_dispatch', { operationId: 'boxless-name', kind: 'click', selector: '#boxless-name-button' });+ return lastMonitorResponse.guard;+ });+ await page.locator('#boxless-name-button span').evaluate(label => { label.textContent = 'Updated action'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated action'), false,+ 'The changed accessible name must not be copied into feedback');+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, 'Changing text in a boxless name descendant must invalidate the prepared action');+ } finally { await browser.close(); }+ });++ test(`${build}: Enter and Space keyboard activations stay attributed to the agent`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const link = document.createElement('a'); link.id = 'keyboard-link'; link.href = '/keyboard-destination';+ link.textContent = 'Open page'; link.addEventListener('click', event => event.preventDefault());+ const button = document.createElement('button'); button.id = 'keyboard-button'; button.type = 'button';+ button.textContent = 'Run action'; button.addEventListener('click', event => event.preventDefault());+ document.body.append(link, button);+ });+ await page.waitForTimeout(300);+ const linkDispatch = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'keyboard-link', tool: 'press_keys', selector: '#keyboard-link' });+ deliver('page_monitor_dispatch', { operationId: 'keyboard-link', kind: 'input', selector: '#keyboard-link' });+ return lastMonitorResponse;+ });+ assert.equal(linkDispatch.ready, true, JSON.stringify(linkDispatch));+ await page.locator('#keyboard-link').press('Enter');+ await page.waitForFunction(() => feedback.some(event => event.source === 'agent' && event.operation === 'click'), null, { timeout: 1000 });+ const linkEvents = await page.evaluate(() => feedback);+ assert.equal(linkEvents.some(event => event.source === 'agent' && event.operation === 'click'+ && event.navigationUrl === 'https://monitor.test/keyboard-destination'), true, JSON.stringify(linkEvents));+ assert.equal(linkEvents.some(event => event.kind === 'click' && event.source === 'user'), false, JSON.stringify(linkEvents));++ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'keyboard-button', tool: 'press_keys', selector: '#keyboard-button' });+ deliver('page_monitor_dispatch', { operationId: 'keyboard-button', kind: 'input', selector: '#keyboard-button' });+ });+ await page.locator('#keyboard-button').press('Space');+ await page.waitForFunction(() => feedback.some(event => event.source === 'agent' && event.operation === 'click'), null, { timeout: 1000 });+ const buttonEvents = await page.evaluate(() => feedback);+ assert.equal(buttonEvents.some(event => event.source === 'agent' && event.operation === 'click'), true, JSON.stringify(buttonEvents));+ assert.equal(buttonEvents.some(event => event.kind === 'click' && event.source === 'user'), false, JSON.stringify(buttonEvents));++ await page.evaluate(() => { feedback = []; });+ await page.locator('#keyboard-button').click();+ await page.waitForFunction(() => feedback.some(event => event.kind === 'click' && event.source === 'user'), null, { timeout: 1000 });+ assert.equal((await page.evaluate(() => feedback)).some(event => event.kind === 'click' && event.source === 'user'), true,+ 'A later pointer click must remain user activity after keyboard attribution');+ } finally { await browser.close(); }+ });+test(`${build}: aria-modal changes invalidate prepared page context`, async () => {const { browser, page } = await fixture(engine, build);try {